diff --git a/.gitattributes b/.gitattributes index a668042419bd9abbe741bb116359a866e1ea2f87..2873ed7751032162eaf89ec3af1e53cfe3868937 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,2 +1,7 @@ * text=auto eol=lf README-header.png filter=lfs diff=lfs merge=lfs -text +apps/android/app/src/main/res/mipmap-xxhdpi/ic_launcher_foreground.png filter=lfs diff=lfs merge=lfs -text +apps/android/app/src/main/res/mipmap-xxxhdpi/ic_launcher_foreground.png filter=lfs diff=lfs merge=lfs -text +apps/ios/Sources/Assets.xcassets/AppIcon.appiconset/icon-1024.png filter=lfs diff=lfs merge=lfs -text +apps/macos/Icon.icon/Assets/openclaw-mac.png filter=lfs diff=lfs merge=lfs -text +apps/macos/Sources/OpenClaw/Resources/OpenClaw.icns filter=lfs diff=lfs merge=lfs -text diff --git a/Swabble/.github/workflows/ci.yml b/Swabble/.github/workflows/ci.yml new file mode 100644 index 0000000000000000000000000000000000000000..aff600f6df00a8da6492515e778958041ec5af99 --- /dev/null +++ b/Swabble/.github/workflows/ci.yml @@ -0,0 +1,54 @@ +name: CI + +on: + push: + branches: [main] + pull_request: + +jobs: + build-and-test: + runs-on: macos-latest + defaults: + run: + shell: bash + working-directory: swabble + steps: + - name: Checkout swabble + uses: actions/checkout@v4 + with: + path: swabble + + - name: Select Xcode 26.1 (prefer 26.1.1) + run: | + set -euo pipefail + # pick the newest installed 26.1.x, fallback to newest 26.x + CANDIDATE="$(ls -d /Applications/Xcode_26.1*.app 2>/dev/null | sort -V | tail -1 || true)" + if [[ -z "$CANDIDATE" ]]; then + CANDIDATE="$(ls -d /Applications/Xcode_26*.app 2>/dev/null | sort -V | tail -1 || true)" + fi + if [[ -z "$CANDIDATE" ]]; then + echo "No Xcode 26.x found on runner" >&2 + exit 1 + fi + echo "Selecting $CANDIDATE" + sudo xcode-select -s "$CANDIDATE" + xcodebuild -version + + - name: Show Swift version + run: swift --version + + - name: Install tooling + run: | + brew update + brew install swiftlint swiftformat + + - name: Format check + run: | + ./scripts/format.sh + git diff --exit-code + + - name: Lint + run: ./scripts/lint.sh + + - name: Test + run: swift test --parallel diff --git a/Swabble/.gitignore b/Swabble/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..e988a5b232b4f2c962c013f2a01ae5e76c0f8670 --- /dev/null +++ b/Swabble/.gitignore @@ -0,0 +1,33 @@ +# macOS +.DS_Store + +# SwiftPM / Build +/.build +/.swiftpm +/DerivedData +xcuserdata/ +*.xcuserstate + +# Editors +/.vscode +.idea/ + +# Xcode artifacts +*.hmap +*.ipa +*.dSYM.zip +*.dSYM + +# Playgrounds +*.xcplayground +playground.xcworkspace +timeline.xctimeline + +# Carthage +Carthage/Build/ + +# fastlane +fastlane/report.xml +fastlane/Preview.html +fastlane/screenshots/**/*.png +fastlane/test_output diff --git a/Swabble/.swiftformat b/Swabble/.swiftformat new file mode 100644 index 0000000000000000000000000000000000000000..2686269a2728eba1a5e65051d3487f9e685716a7 --- /dev/null +++ b/Swabble/.swiftformat @@ -0,0 +1,8 @@ +--swiftversion 6.2 +--indent 4 +--maxwidth 120 +--wraparguments before-first +--wrapcollections before-first +--stripunusedargs closure-only +--self remove +--header "" diff --git a/Swabble/.swiftlint.yml b/Swabble/.swiftlint.yml new file mode 100644 index 0000000000000000000000000000000000000000..f63ff5dbb18d53fa56f0cb25bcb10401587fad5a --- /dev/null +++ b/Swabble/.swiftlint.yml @@ -0,0 +1,43 @@ +# SwiftLint for swabble +included: + - Sources +excluded: + - .build + - DerivedData + - "**/.swiftpm" + - "**/.build" + - "**/DerivedData" + - "**/.DS_Store" +opt_in_rules: + - array_init + - closure_spacing + - explicit_init + - fatal_error_message + - first_where + - joined_default_parameter + - last_where + - literal_expression_end_indentation + - multiline_arguments + - multiline_parameters + - operator_usage_whitespace + - redundant_nil_coalescing + - sorted_first_last + - switch_case_alignment + - vertical_parameter_alignment_on_call + - vertical_whitespace_opening_braces + - vertical_whitespace_closing_braces + +disabled_rules: + - trailing_whitespace + - trailing_newline + - indentation_width + - identifier_name + - explicit_self + - file_header + - todo + +line_length: + warning: 140 + error: 180 + +reporter: "xcode" diff --git a/Swabble/CHANGELOG.md b/Swabble/CHANGELOG.md new file mode 100644 index 0000000000000000000000000000000000000000..e8f2ad60d857e550e692e357cd30c5add66429c8 --- /dev/null +++ b/Swabble/CHANGELOG.md @@ -0,0 +1,11 @@ +# Changelog + +## 0.2.0 — 2025-12-23 + +### Highlights +- Added `SwabbleKit` (multi-platform wake-word gate utilities with segment-aware gap detection). +- Swabble package now supports iOS + macOS consumers; CLI remains macOS 26-only. + +### Changes +- CLI wake-word matching/stripping routed through `SwabbleKit` helpers. +- Speech pipeline types now explicitly gated to macOS 26 / iOS 26 availability. diff --git a/Swabble/LICENSE b/Swabble/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..f7b526698bb7ed2d26d96c49f2f32234c88f69bc --- /dev/null +++ b/Swabble/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2025 Peter Steinberger + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/Swabble/Package.resolved b/Swabble/Package.resolved new file mode 100644 index 0000000000000000000000000000000000000000..f52a51fbe534da93b756d59232ff72464d391d2b --- /dev/null +++ b/Swabble/Package.resolved @@ -0,0 +1,69 @@ +{ + "originHash" : "24a723309d7a0039d3df3051106f77ac1ed7068a02508e3a6804e41d757e6c72", + "pins" : [ + { + "identity" : "commander", + "kind" : "remoteSourceControl", + "location" : "https://github.com/steipete/Commander.git", + "state" : { + "revision" : "9e349575c8e3c6745e81fe19e5bb5efa01b078ce", + "version" : "0.2.1" + } + }, + { + "identity" : "elevenlabskit", + "kind" : "remoteSourceControl", + "location" : "https://github.com/steipete/ElevenLabsKit", + "state" : { + "revision" : "7e3c948d8340abe3977014f3de020edf221e9269", + "version" : "0.1.0" + } + }, + { + "identity" : "swift-concurrency-extras", + "kind" : "remoteSourceControl", + "location" : "https://github.com/pointfreeco/swift-concurrency-extras", + "state" : { + "revision" : "5a3825302b1a0d744183200915a47b508c828e6f", + "version" : "1.3.2" + } + }, + { + "identity" : "swift-syntax", + "kind" : "remoteSourceControl", + "location" : "https://github.com/swiftlang/swift-syntax.git", + "state" : { + "revision" : "0687f71944021d616d34d922343dcef086855920", + "version" : "600.0.1" + } + }, + { + "identity" : "swift-testing", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-testing", + "state" : { + "revision" : "399f76dcd91e4c688ca2301fa24a8cc6d9927211", + "version" : "0.99.0" + } + }, + { + "identity" : "swiftui-math", + "kind" : "remoteSourceControl", + "location" : "https://github.com/gonzalezreal/swiftui-math", + "state" : { + "revision" : "0b5c2cfaaec8d6193db206f675048eeb5ce95f71", + "version" : "0.1.0" + } + }, + { + "identity" : "textual", + "kind" : "remoteSourceControl", + "location" : "https://github.com/gonzalezreal/textual", + "state" : { + "revision" : "5b06b811c0f5313b6b84bbef98c635a630638c38", + "version" : "0.3.1" + } + } + ], + "version" : 3 +} diff --git a/Swabble/Package.swift b/Swabble/Package.swift new file mode 100644 index 0000000000000000000000000000000000000000..9f5a000361921fcdc2ce50ce131f43b827d96cb2 --- /dev/null +++ b/Swabble/Package.swift @@ -0,0 +1,55 @@ +// swift-tools-version: 6.2 +import PackageDescription + +let package = Package( + name: "swabble", + platforms: [ + .macOS(.v15), + .iOS(.v17), + ], + products: [ + .library(name: "Swabble", targets: ["Swabble"]), + .library(name: "SwabbleKit", targets: ["SwabbleKit"]), + .executable(name: "swabble", targets: ["SwabbleCLI"]), + ], + dependencies: [ + .package(url: "https://github.com/steipete/Commander.git", exact: "0.2.1"), + .package(url: "https://github.com/apple/swift-testing", from: "0.99.0"), + ], + targets: [ + .target( + name: "Swabble", + path: "Sources/SwabbleCore", + swiftSettings: []), + .target( + name: "SwabbleKit", + path: "Sources/SwabbleKit", + swiftSettings: [ + .enableUpcomingFeature("StrictConcurrency"), + ]), + .executableTarget( + name: "SwabbleCLI", + dependencies: [ + "Swabble", + "SwabbleKit", + .product(name: "Commander", package: "Commander"), + ], + path: "Sources/swabble"), + .testTarget( + name: "SwabbleKitTests", + dependencies: [ + "SwabbleKit", + .product(name: "Testing", package: "swift-testing"), + ], + swiftSettings: [ + .enableUpcomingFeature("StrictConcurrency"), + .enableExperimentalFeature("SwiftTesting"), + ]), + .testTarget( + name: "swabbleTests", + dependencies: [ + "Swabble", + .product(name: "Testing", package: "swift-testing"), + ]), + ], + swiftLanguageModes: [.v6]) diff --git a/Swabble/README.md b/Swabble/README.md new file mode 100644 index 0000000000000000000000000000000000000000..bf6dc3dc8bd026bd405df40ce00eb5443ec33a6e --- /dev/null +++ b/Swabble/README.md @@ -0,0 +1,111 @@ +# 🎙️ swabble — Speech.framework wake-word hook daemon (macOS 26) + +swabble is a Swift 6.2 wake-word hook daemon. The CLI targets macOS 26 (SpeechAnalyzer + SpeechTranscriber). The shared `SwabbleKit` target is multi-platform and exposes wake-word gating utilities for iOS/macOS apps. + +- **Local-only**: Speech.framework on-device models; zero network usage. +- **Wake word**: Default `clawd` (aliases `claude`), optional `--no-wake` bypass. +- **SwabbleKit**: Shared wake gate utilities (gap-based gating when you provide speech segments). +- **Hooks**: Run any command with prefix/env, cooldown, min_chars, timeout. +- **Services**: launchd helper stubs for start/stop/install. +- **File transcribe**: TXT or SRT with time ranges (using AttributedString splits). + +## Quick start +```bash +# Install deps +brew install swiftformat swiftlint + +# Build +swift build + +# Write default config (~/.config/swabble/config.json) +swift run swabble setup + +# Run foreground daemon +swift run swabble serve + +# Test your hook +swift run swabble test-hook "hello world" + +# Transcribe a file to SRT +swift run swabble transcribe /path/to/audio.m4a --format srt --output out.srt +``` + +## Use as a library +Add swabble as a SwiftPM dependency and import the `Swabble` or `SwabbleKit` product: + +```swift +// Package.swift +dependencies: [ + .package(url: "https://github.com/steipete/swabble.git", branch: "main"), +], +targets: [ + .target(name: "MyApp", dependencies: [ + .product(name: "Swabble", package: "swabble"), // Speech pipeline (macOS 26+ / iOS 26+) + .product(name: "SwabbleKit", package: "swabble"), // Wake-word gate utilities (iOS 17+ / macOS 15+) + ]), +] +``` + +## CLI +- `serve` — foreground loop (mic → wake → hook) +- `transcribe ` — offline transcription (txt|srt) +- `test-hook "text"` — invoke configured hook +- `mic list|set ` — enumerate/select input device +- `setup` — write default config JSON +- `doctor` — check Speech auth & device availability +- `health` — prints `ok` +- `tail-log` — last 10 transcripts +- `status` — show wake state + recent transcripts +- `service install|uninstall|status` — user launchd plist (stub: prints launchctl commands) +- `start|stop|restart` — placeholders until full launchd wiring + +All commands accept Commander runtime flags (`-v/--verbose`, `--json-output`, `--log-level`), plus `--config` where applicable. + +## Config +`~/.config/swabble/config.json` (auto-created by `setup`): +```json +{ + "audio": {"deviceName": "", "deviceIndex": -1, "sampleRate": 16000, "channels": 1}, + "wake": {"enabled": true, "word": "clawd", "aliases": ["claude"]}, + "hook": { + "command": "", + "args": [], + "prefix": "Voice swabble from ${hostname}: ", + "cooldownSeconds": 1, + "minCharacters": 24, + "timeoutSeconds": 5, + "env": {} + }, + "logging": {"level": "info", "format": "text"}, + "transcripts": {"enabled": true, "maxEntries": 50}, + "speech": {"localeIdentifier": "en_US", "etiquetteReplacements": false} +} +``` + +- Config path override: `--config /path/to/config.json` on relevant commands. +- Transcripts persist to `~/Library/Application Support/swabble/transcripts.log`. + +## Hook protocol +When a wake-gated transcript passes min_chars & cooldown, swabble runs: +``` + "" +``` +Environment variables: +- `SWABBLE_TEXT` — stripped transcript (wake word removed) +- `SWABBLE_PREFIX` — rendered prefix (hostname substituted) +- plus any `hook.env` key/values + +## Speech pipeline +- `AVAudioEngine` tap → `BufferConverter` → `AnalyzerInput` → `SpeechAnalyzer` with a `SpeechTranscriber` module. +- Requests volatile + final results; the CLI uses text-only wake gating today. +- Authorization requested at first start; requires macOS 26 + new Speech.framework APIs. + +## Development +- Format: `./scripts/format.sh` (uses local `.swiftformat`) +- Lint: `./scripts/lint.sh` (uses local `.swiftlint.yml`) +- Tests: `swift test` (uses swift-testing package) + +## Roadmap +- launchd control (load/bootout, PID + status socket) +- JSON logging + PII redaction toggle +- Stronger wake-word detection and control socket status/health diff --git a/Swabble/Sources/SwabbleCore/Config/Config.swift b/Swabble/Sources/SwabbleCore/Config/Config.swift new file mode 100644 index 0000000000000000000000000000000000000000..4dc9d4668c029ce573a31f00bb6b5def31dbf424 --- /dev/null +++ b/Swabble/Sources/SwabbleCore/Config/Config.swift @@ -0,0 +1,77 @@ +import Foundation + +public struct SwabbleConfig: Codable, Sendable { + public struct Audio: Codable, Sendable { + public var deviceName: String = "" + public var deviceIndex: Int = -1 + public var sampleRate: Double = 16000 + public var channels: Int = 1 + } + + public struct Wake: Codable, Sendable { + public var enabled: Bool = true + public var word: String = "clawd" + public var aliases: [String] = ["claude"] + } + + public struct Hook: Codable, Sendable { + public var command: String = "" + public var args: [String] = [] + public var prefix: String = "Voice swabble from ${hostname}: " + public var cooldownSeconds: Double = 1 + public var minCharacters: Int = 24 + public var timeoutSeconds: Double = 5 + public var env: [String: String] = [:] + } + + public struct Logging: Codable, Sendable { + public var level: String = "info" + public var format: String = "text" // text|json placeholder + } + + public struct Transcripts: Codable, Sendable { + public var enabled: Bool = true + public var maxEntries: Int = 50 + } + + public struct Speech: Codable, Sendable { + public var localeIdentifier: String = Locale.current.identifier + public var etiquetteReplacements: Bool = false + } + + public var audio = Audio() + public var wake = Wake() + public var hook = Hook() + public var logging = Logging() + public var transcripts = Transcripts() + public var speech = Speech() + + public static let defaultPath = FileManager.default + .homeDirectoryForCurrentUser + .appendingPathComponent(".config/swabble/config.json") + + public init() {} +} + +public enum ConfigError: Error { + case missingConfig +} + +public enum ConfigLoader { + public static func load(at path: URL?) throws -> SwabbleConfig { + let url = path ?? SwabbleConfig.defaultPath + if !FileManager.default.fileExists(atPath: url.path) { + throw ConfigError.missingConfig + } + let data = try Data(contentsOf: url) + return try JSONDecoder().decode(SwabbleConfig.self, from: data) + } + + public static func save(_ config: SwabbleConfig, at path: URL?) throws { + let url = path ?? SwabbleConfig.defaultPath + let dir = url.deletingLastPathComponent() + try FileManager.default.createDirectory(at: dir, withIntermediateDirectories: true) + let data = try JSONEncoder().encode(config) + try data.write(to: url) + } +} diff --git a/Swabble/Sources/SwabbleCore/Hooks/HookExecutor.swift b/Swabble/Sources/SwabbleCore/Hooks/HookExecutor.swift new file mode 100644 index 0000000000000000000000000000000000000000..dd59c43bb58dce51dc31f0a2966f456b6417ee50 --- /dev/null +++ b/Swabble/Sources/SwabbleCore/Hooks/HookExecutor.swift @@ -0,0 +1,75 @@ +import Foundation + +public struct HookJob: Sendable { + public let text: String + public let timestamp: Date + + public init(text: String, timestamp: Date) { + self.text = text + self.timestamp = timestamp + } +} + +public actor HookExecutor { + private let config: SwabbleConfig + private var lastRun: Date? + private let hostname: String + + public init(config: SwabbleConfig) { + self.config = config + hostname = Host.current().localizedName ?? "host" + } + + public func shouldRun() -> Bool { + guard config.hook.cooldownSeconds > 0 else { return true } + if let lastRun, Date().timeIntervalSince(lastRun) < config.hook.cooldownSeconds { + return false + } + return true + } + + public func run(job: HookJob) async throws { + guard shouldRun() else { return } + guard !config.hook.command.isEmpty else { throw NSError( + domain: "Hook", + code: 1, + userInfo: [NSLocalizedDescriptionKey: "hook command not set"]) } + + let prefix = config.hook.prefix.replacingOccurrences(of: "${hostname}", with: hostname) + let payload = prefix + job.text + + let process = Process() + process.executableURL = URL(fileURLWithPath: config.hook.command) + process.arguments = config.hook.args + [payload] + + var env = ProcessInfo.processInfo.environment + env["SWABBLE_TEXT"] = job.text + env["SWABBLE_PREFIX"] = prefix + for (k, v) in config.hook.env { + env[k] = v + } + process.environment = env + + let pipe = Pipe() + process.standardOutput = pipe + process.standardError = pipe + + try process.run() + + let timeoutNanos = UInt64(max(config.hook.timeoutSeconds, 0.1) * 1_000_000_000) + try await withThrowingTaskGroup(of: Void.self) { group in + group.addTask { + process.waitUntilExit() + } + group.addTask { + try await Task.sleep(nanoseconds: timeoutNanos) + if process.isRunning { + process.terminate() + } + } + try await group.next() + group.cancelAll() + } + lastRun = Date() + } +} diff --git a/Swabble/Sources/SwabbleCore/Speech/BufferConverter.swift b/Swabble/Sources/SwabbleCore/Speech/BufferConverter.swift new file mode 100644 index 0000000000000000000000000000000000000000..e6d7dc993badd526fa7bcba60cb7de7023432c56 --- /dev/null +++ b/Swabble/Sources/SwabbleCore/Speech/BufferConverter.swift @@ -0,0 +1,50 @@ +@preconcurrency import AVFoundation +import Foundation + +final class BufferConverter { + private final class Box: @unchecked Sendable { var value: T; init(_ value: T) { self.value = value } } + enum ConverterError: Swift.Error { + case failedToCreateConverter + case failedToCreateConversionBuffer + case conversionFailed(NSError?) + } + + private var converter: AVAudioConverter? + + func convert(_ buffer: AVAudioPCMBuffer, to format: AVAudioFormat) throws -> AVAudioPCMBuffer { + let inputFormat = buffer.format + if inputFormat == format { + return buffer + } + if converter == nil || converter?.outputFormat != format { + converter = AVAudioConverter(from: inputFormat, to: format) + converter?.primeMethod = .none + } + guard let converter else { throw ConverterError.failedToCreateConverter } + + let sampleRateRatio = converter.outputFormat.sampleRate / converter.inputFormat.sampleRate + let scaledInputFrameLength = Double(buffer.frameLength) * sampleRateRatio + let frameCapacity = AVAudioFrameCount(scaledInputFrameLength.rounded(.up)) + guard let conversionBuffer = AVAudioPCMBuffer(pcmFormat: converter.outputFormat, frameCapacity: frameCapacity) + else { + throw ConverterError.failedToCreateConversionBuffer + } + + var nsError: NSError? + let consumed = Box(false) + let inputBuffer = buffer + let status = converter.convert(to: conversionBuffer, error: &nsError) { _, statusPtr in + if consumed.value { + statusPtr.pointee = .noDataNow + return nil + } + consumed.value = true + statusPtr.pointee = .haveData + return inputBuffer + } + if status == .error { + throw ConverterError.conversionFailed(nsError) + } + return conversionBuffer + } +} diff --git a/Swabble/Sources/SwabbleCore/Speech/SpeechPipeline.swift b/Swabble/Sources/SwabbleCore/Speech/SpeechPipeline.swift new file mode 100644 index 0000000000000000000000000000000000000000..014b174da7bf9aa580dbdbd2288f59c02b6c7572 --- /dev/null +++ b/Swabble/Sources/SwabbleCore/Speech/SpeechPipeline.swift @@ -0,0 +1,114 @@ +import AVFoundation +import Foundation +import Speech + +@available(macOS 26.0, iOS 26.0, *) +public struct SpeechSegment: Sendable { + public let text: String + public let isFinal: Bool +} + +@available(macOS 26.0, iOS 26.0, *) +public enum SpeechPipelineError: Error { + case authorizationDenied + case analyzerFormatUnavailable + case transcriberUnavailable +} + +/// Live microphone → SpeechAnalyzer → SpeechTranscriber pipeline. +@available(macOS 26.0, iOS 26.0, *) +public actor SpeechPipeline { + private struct UnsafeBuffer: @unchecked Sendable { let buffer: AVAudioPCMBuffer } + + private var engine = AVAudioEngine() + private var transcriber: SpeechTranscriber? + private var analyzer: SpeechAnalyzer? + private var inputContinuation: AsyncStream.Continuation? + private var resultTask: Task? + private let converter = BufferConverter() + + public init() {} + + public func start(localeIdentifier: String, etiquette: Bool) async throws -> AsyncStream { + let auth = await requestAuthorizationIfNeeded() + guard auth == .authorized else { throw SpeechPipelineError.authorizationDenied } + + let transcriberModule = SpeechTranscriber( + locale: Locale(identifier: localeIdentifier), + transcriptionOptions: etiquette ? [.etiquetteReplacements] : [], + reportingOptions: [.volatileResults], + attributeOptions: []) + transcriber = transcriberModule + + guard let analyzerFormat = await SpeechAnalyzer.bestAvailableAudioFormat(compatibleWith: [transcriberModule]) + else { + throw SpeechPipelineError.analyzerFormatUnavailable + } + + analyzer = SpeechAnalyzer(modules: [transcriberModule]) + let (stream, continuation) = AsyncStream.makeStream() + inputContinuation = continuation + + let inputNode = engine.inputNode + let inputFormat = inputNode.outputFormat(forBus: 0) + inputNode.removeTap(onBus: 0) + inputNode.installTap(onBus: 0, bufferSize: 2048, format: inputFormat) { [weak self] buffer, _ in + guard let self else { return } + let boxed = UnsafeBuffer(buffer: buffer) + Task { await self.handleBuffer(boxed.buffer, targetFormat: analyzerFormat) } + } + + engine.prepare() + try engine.start() + try await analyzer?.start(inputSequence: stream) + + guard let transcriberForStream = transcriber else { + throw SpeechPipelineError.transcriberUnavailable + } + + return AsyncStream { continuation in + self.resultTask = Task { + do { + for try await result in transcriberForStream.results { + let seg = SpeechSegment(text: String(result.text.characters), isFinal: result.isFinal) + continuation.yield(seg) + } + } catch { + // swallow errors and finish + } + continuation.finish() + } + continuation.onTermination = { _ in + Task { await self.stop() } + } + } + } + + public func stop() async { + resultTask?.cancel() + inputContinuation?.finish() + engine.inputNode.removeTap(onBus: 0) + engine.stop() + try? await analyzer?.finalizeAndFinishThroughEndOfInput() + } + + private func handleBuffer(_ buffer: AVAudioPCMBuffer, targetFormat: AVAudioFormat) async { + do { + let converted = try converter.convert(buffer, to: targetFormat) + let input = AnalyzerInput(buffer: converted) + inputContinuation?.yield(input) + } catch { + // drop on conversion failure + } + } + + private func requestAuthorizationIfNeeded() async -> SFSpeechRecognizerAuthorizationStatus { + let current = SFSpeechRecognizer.authorizationStatus() + guard current == .notDetermined else { return current } + return await withCheckedContinuation { continuation in + SFSpeechRecognizer.requestAuthorization { status in + continuation.resume(returning: status) + } + } + } +} diff --git a/Swabble/Sources/SwabbleCore/Support/AttributedString+Sentences.swift b/Swabble/Sources/SwabbleCore/Support/AttributedString+Sentences.swift new file mode 100644 index 0000000000000000000000000000000000000000..e2de6fdfce58e3ff5024b629542724fbb1ebd905 --- /dev/null +++ b/Swabble/Sources/SwabbleCore/Support/AttributedString+Sentences.swift @@ -0,0 +1,62 @@ +import CoreMedia +import Foundation +import NaturalLanguage + +extension AttributedString { + public func sentences(maxLength: Int? = nil) -> [AttributedString] { + let tokenizer = NLTokenizer(unit: .sentence) + let string = String(characters) + tokenizer.string = string + let sentenceRanges = tokenizer.tokens(for: string.startIndex.. maxLength else { + return [sentenceRange] + } + + let wordTokenizer = NLTokenizer(unit: .word) + wordTokenizer.string = string + var wordRanges = wordTokenizer.tokens(for: sentenceStringRange).map { + AttributedString.Index($0.lowerBound, within: self)! + ..< + AttributedString.Index($0.upperBound, within: self)! + } + guard !wordRanges.isEmpty else { return [sentenceRange] } + wordRanges[0] = sentenceRange.lowerBound..] = [] + for wordRange in wordRanges { + if let lastRange = ranges.last, + self[lastRange].characters.count + self[wordRange].characters.count <= maxLength { + ranges[ranges.count - 1] = lastRange.lowerBound.. Bool { lhs.rank < rhs.rank } +} + +public struct Logger: Sendable { + public let level: LogLevel + + public init(level: LogLevel) { self.level = level } + + public func log(_ level: LogLevel, _ message: String) { + guard level >= self.level else { return } + let ts = ISO8601DateFormatter().string(from: Date()) + print("[\(level.rawValue.uppercased())] \(ts) | \(message)") + } + + public func trace(_ msg: String) { log(.trace, msg) } + public func debug(_ msg: String) { log(.debug, msg) } + public func info(_ msg: String) { log(.info, msg) } + public func warn(_ msg: String) { log(.warn, msg) } + public func error(_ msg: String) { log(.error, msg) } +} + +extension LogLevel { + public init?(configValue: String) { + self.init(rawValue: configValue.lowercased()) + } +} diff --git a/Swabble/Sources/SwabbleCore/Support/OutputFormat.swift b/Swabble/Sources/SwabbleCore/Support/OutputFormat.swift new file mode 100644 index 0000000000000000000000000000000000000000..84047c7284b295ff4607f9734a6444d8f4d50d31 --- /dev/null +++ b/Swabble/Sources/SwabbleCore/Support/OutputFormat.swift @@ -0,0 +1,45 @@ +import CoreMedia +import Foundation + +public enum OutputFormat: String { + case txt + case srt + + public var needsAudioTimeRange: Bool { + switch self { + case .srt: true + default: false + } + } + + public func text(for transcript: AttributedString, maxLength: Int) -> String { + switch self { + case .txt: + return String(transcript.characters) + case .srt: + func format(_ timeInterval: TimeInterval) -> String { + let ms = Int(timeInterval.truncatingRemainder(dividingBy: 1) * 1000) + let s = Int(timeInterval) % 60 + let m = (Int(timeInterval) / 60) % 60 + let h = Int(timeInterval) / 60 / 60 + return String(format: "%0.2d:%0.2d:%0.2d,%0.3d", h, m, s, ms) + } + + return transcript.sentences(maxLength: maxLength).compactMap { (sentence: AttributedString) -> ( + CMTimeRange, + String)? in + guard let timeRange = sentence.audioTimeRange else { return nil } + return (timeRange, String(sentence.characters)) + }.enumerated().map { index, run in + let (timeRange, text) = run + return """ + + \(index + 1) + \(format(timeRange.start.seconds)) --> \(format(timeRange.end.seconds)) + \(text.trimmingCharacters(in: .whitespacesAndNewlines)) + + """ + }.joined().trimmingCharacters(in: .whitespacesAndNewlines) + } + } +} diff --git a/Swabble/Sources/SwabbleCore/Support/TranscriptsStore.swift b/Swabble/Sources/SwabbleCore/Support/TranscriptsStore.swift new file mode 100644 index 0000000000000000000000000000000000000000..4f91d052e6a64ca9c1049b26cca5c9ac88806522 --- /dev/null +++ b/Swabble/Sources/SwabbleCore/Support/TranscriptsStore.swift @@ -0,0 +1,45 @@ +import Foundation + +public actor TranscriptsStore { + public static let shared = TranscriptsStore() + + private var entries: [String] = [] + private let limit = 100 + private let fileURL: URL + + public init() { + let dir = FileManager.default.homeDirectoryForCurrentUser + .appendingPathComponent("Library/Application Support/swabble", isDirectory: true) + try? FileManager.default.createDirectory(at: dir, withIntermediateDirectories: true) + fileURL = dir.appendingPathComponent("transcripts.log") + if let data = try? Data(contentsOf: fileURL), + let text = String(data: data, encoding: .utf8) { + entries = text.split(separator: "\n").map(String.init).suffix(limit) + } + } + + public func append(text: String) { + entries.append(text) + if entries.count > limit { + entries.removeFirst(entries.count - limit) + } + let body = entries.joined(separator: "\n") + try? body.write(to: fileURL, atomically: false, encoding: .utf8) + } + + public func latest() -> [String] { entries } +} + +extension String { + private func appendLine(to url: URL) throws { + let data = (self + "\n").data(using: .utf8) ?? Data() + if FileManager.default.fileExists(atPath: url.path) { + let handle = try FileHandle(forWritingTo: url) + try handle.seekToEnd() + try handle.write(contentsOf: data) + try handle.close() + } else { + try data.write(to: url) + } + } +} diff --git a/Swabble/Sources/SwabbleKit/WakeWordGate.swift b/Swabble/Sources/SwabbleKit/WakeWordGate.swift new file mode 100644 index 0000000000000000000000000000000000000000..27c952a8d1b659be286e7f4a6c40ed0a4390555f --- /dev/null +++ b/Swabble/Sources/SwabbleKit/WakeWordGate.swift @@ -0,0 +1,197 @@ +import Foundation + +public struct WakeWordSegment: Sendable, Equatable { + public let text: String + public let start: TimeInterval + public let duration: TimeInterval + public let range: Range? + + public init(text: String, start: TimeInterval, duration: TimeInterval, range: Range? = nil) { + self.text = text + self.start = start + self.duration = duration + self.range = range + } + + public var end: TimeInterval { start + duration } +} + +public struct WakeWordGateConfig: Sendable, Equatable { + public var triggers: [String] + public var minPostTriggerGap: TimeInterval + public var minCommandLength: Int + + public init( + triggers: [String], + minPostTriggerGap: TimeInterval = 0.45, + minCommandLength: Int = 1) { + self.triggers = triggers + self.minPostTriggerGap = minPostTriggerGap + self.minCommandLength = minCommandLength + } +} + +public struct WakeWordGateMatch: Sendable, Equatable { + public let triggerEndTime: TimeInterval + public let postGap: TimeInterval + public let command: String + + public init(triggerEndTime: TimeInterval, postGap: TimeInterval, command: String) { + self.triggerEndTime = triggerEndTime + self.postGap = postGap + self.command = command + } +} + +public enum WakeWordGate { + private struct Token { + let normalized: String + let start: TimeInterval + let end: TimeInterval + let range: Range? + let text: String + } + + private struct TriggerTokens { + let tokens: [String] + } + + private struct MatchCandidate { + let index: Int + let triggerEnd: TimeInterval + let gap: TimeInterval + } + + public static func match( + transcript: String, + segments: [WakeWordSegment], + config: WakeWordGateConfig) + -> WakeWordGateMatch? { + let triggerTokens = normalizeTriggers(config.triggers) + guard !triggerTokens.isEmpty else { return nil } + + let tokens = normalizeSegments(segments) + guard !tokens.isEmpty else { return nil } + + var best: MatchCandidate? + + for trigger in triggerTokens { + let count = trigger.tokens.count + guard count > 0, tokens.count > count else { continue } + for i in 0...(tokens.count - count - 1) { + let matched = (0..= config.minCommandLength else { return nil } + return WakeWordGateMatch(triggerEndTime: best.triggerEnd, postGap: best.gap, command: command) + } + + public static func commandText( + transcript: String, + segments: [WakeWordSegment], + triggerEndTime: TimeInterval) + -> String { + let threshold = triggerEndTime + 0.001 + for segment in segments where segment.start >= threshold { + if normalizeToken(segment.text).isEmpty { continue } + if let range = segment.range { + let slice = transcript[range.lowerBound...] + return String(slice).trimmingCharacters(in: Self.whitespaceAndPunctuation) + } + break + } + + let text = segments + .filter { $0.start >= threshold && !normalizeToken($0.text).isEmpty } + .map(\.text) + .joined(separator: " ") + return text.trimmingCharacters(in: Self.whitespaceAndPunctuation) + } + + public static func matchesTextOnly(text: String, triggers: [String]) -> Bool { + guard !text.isEmpty else { return false } + let normalized = text.lowercased() + for trigger in triggers { + let token = trigger.trimmingCharacters(in: whitespaceAndPunctuation).lowercased() + if token.isEmpty { continue } + if normalized.contains(token) { return true } + } + return false + } + + public static func stripWake(text: String, triggers: [String]) -> String { + var out = text + for trigger in triggers { + let token = trigger.trimmingCharacters(in: whitespaceAndPunctuation) + guard !token.isEmpty else { continue } + out = out.replacingOccurrences(of: token, with: "", options: [.caseInsensitive]) + } + return out.trimmingCharacters(in: whitespaceAndPunctuation) + } + + private static func normalizeTriggers(_ triggers: [String]) -> [TriggerTokens] { + var output: [TriggerTokens] = [] + for trigger in triggers { + let tokens = trigger + .split(whereSeparator: { $0.isWhitespace }) + .map { normalizeToken(String($0)) } + .filter { !$0.isEmpty } + if tokens.isEmpty { continue } + output.append(TriggerTokens(tokens: tokens)) + } + return output + } + + private static func normalizeSegments(_ segments: [WakeWordSegment]) -> [Token] { + segments.compactMap { segment in + let normalized = normalizeToken(segment.text) + guard !normalized.isEmpty else { return nil } + return Token( + normalized: normalized, + start: segment.start, + end: segment.end, + range: segment.range, + text: segment.text) + } + } + + private static func normalizeToken(_ token: String) -> String { + token + .trimmingCharacters(in: whitespaceAndPunctuation) + .lowercased() + } + + private static let whitespaceAndPunctuation = CharacterSet.whitespacesAndNewlines + .union(.punctuationCharacters) +} + +#if canImport(Speech) +import Speech + +public enum WakeWordSpeechSegments { + public static func from(transcription: SFTranscription, transcript: String) -> [WakeWordSegment] { + transcription.segments.map { segment in + let range = Range(segment.substringRange, in: transcript) + return WakeWordSegment( + text: segment.substring, + start: segment.timestamp, + duration: segment.duration, + range: range) + } + } +} +#endif diff --git a/Swabble/Sources/swabble/CLI/CLIRegistry.swift b/Swabble/Sources/swabble/CLI/CLIRegistry.swift new file mode 100644 index 0000000000000000000000000000000000000000..c47a9864f9aa8b97419a6896945e434f2c7fbceb --- /dev/null +++ b/Swabble/Sources/swabble/CLI/CLIRegistry.swift @@ -0,0 +1,71 @@ +import Commander +import Foundation + +@available(macOS 26.0, *) +@MainActor +enum CLIRegistry { + static var descriptors: [CommandDescriptor] { + let serveDesc = descriptor(for: ServeCommand.self) + let transcribeDesc = descriptor(for: TranscribeCommand.self) + let testHookDesc = descriptor(for: TestHookCommand.self) + let micList = descriptor(for: MicList.self) + let micSet = descriptor(for: MicSet.self) + let micRoot = CommandDescriptor( + name: "mic", + abstract: "Microphone management", + discussion: nil, + signature: CommandSignature(), + subcommands: [micList, micSet]) + let serviceRoot = CommandDescriptor( + name: "service", + abstract: "launchd helper", + discussion: nil, + signature: CommandSignature(), + subcommands: [ + descriptor(for: ServiceInstall.self), + descriptor(for: ServiceUninstall.self), + descriptor(for: ServiceStatus.self) + ]) + let doctorDesc = descriptor(for: DoctorCommand.self) + let setupDesc = descriptor(for: SetupCommand.self) + let healthDesc = descriptor(for: HealthCommand.self) + let tailLogDesc = descriptor(for: TailLogCommand.self) + let startDesc = descriptor(for: StartCommand.self) + let stopDesc = descriptor(for: StopCommand.self) + let restartDesc = descriptor(for: RestartCommand.self) + let statusDesc = descriptor(for: StatusCommand.self) + + let rootSignature = CommandSignature().withStandardRuntimeFlags() + let root = CommandDescriptor( + name: "swabble", + abstract: "Speech hook daemon", + discussion: "Local wake-word → SpeechTranscriber → hook", + signature: rootSignature, + subcommands: [ + serveDesc, + transcribeDesc, + testHookDesc, + micRoot, + serviceRoot, + doctorDesc, + setupDesc, + healthDesc, + tailLogDesc, + startDesc, + stopDesc, + restartDesc, + statusDesc + ]) + return [root] + } + + private static func descriptor(for type: any ParsableCommand.Type) -> CommandDescriptor { + let sig = CommandSignature.describe(type.init()).withStandardRuntimeFlags() + return CommandDescriptor( + name: type.commandDescription.commandName ?? "", + abstract: type.commandDescription.abstract, + discussion: type.commandDescription.discussion, + signature: sig, + subcommands: []) + } +} diff --git a/Swabble/Sources/swabble/Commands/DoctorCommand.swift b/Swabble/Sources/swabble/Commands/DoctorCommand.swift new file mode 100644 index 0000000000000000000000000000000000000000..ec6c84ad44a0d3a500a9776427fd66f29a5a26eb --- /dev/null +++ b/Swabble/Sources/swabble/Commands/DoctorCommand.swift @@ -0,0 +1,37 @@ +import Commander +import Foundation +import Speech +import Swabble + +@MainActor +struct DoctorCommand: ParsableCommand { + static var commandDescription: CommandDescription { + CommandDescription(commandName: "doctor", abstract: "Check Speech permission and config") + } + + @Option(name: .long("config"), help: "Path to config JSON") var configPath: String? + + init() {} + init(parsed: ParsedValues) { + self.init() + if let cfg = parsed.options["config"]?.last { configPath = cfg } + } + + mutating func run() async throws { + let auth = await SFSpeechRecognizer.authorizationStatus() + print("Speech auth: \(auth)") + do { + _ = try ConfigLoader.load(at: configURL) + print("Config: OK") + } catch { + print("Config missing or invalid; run setup") + } + let session = AVCaptureDevice.DiscoverySession( + deviceTypes: [.microphone, .external], + mediaType: .audio, + position: .unspecified) + print("Mics found: \(session.devices.count)") + } + + private var configURL: URL? { configPath.map { URL(fileURLWithPath: $0) } } +} diff --git a/Swabble/Sources/swabble/Commands/HealthCommand.swift b/Swabble/Sources/swabble/Commands/HealthCommand.swift new file mode 100644 index 0000000000000000000000000000000000000000..b3db452868dcd138ee6532a494a233ce880b0e3a --- /dev/null +++ b/Swabble/Sources/swabble/Commands/HealthCommand.swift @@ -0,0 +1,16 @@ +import Commander +import Foundation + +@MainActor +struct HealthCommand: ParsableCommand { + static var commandDescription: CommandDescription { + CommandDescription(commandName: "health", abstract: "Health probe") + } + + init() {} + init(parsed: ParsedValues) {} + + mutating func run() async throws { + print("ok") + } +} diff --git a/Swabble/Sources/swabble/Commands/MicCommands.swift b/Swabble/Sources/swabble/Commands/MicCommands.swift new file mode 100644 index 0000000000000000000000000000000000000000..6430c86d529ba12c9c720adf5eb3f263a3d8f3cb --- /dev/null +++ b/Swabble/Sources/swabble/Commands/MicCommands.swift @@ -0,0 +1,62 @@ +import AVFoundation +import Commander +import Foundation +import Swabble + +@MainActor +struct MicCommand: ParsableCommand { + static var commandDescription: CommandDescription { + CommandDescription( + commandName: "mic", + abstract: "Microphone management", + subcommands: [MicList.self, MicSet.self]) + } +} + +@MainActor +struct MicList: ParsableCommand { + static var commandDescription: CommandDescription { + CommandDescription(commandName: "list", abstract: "List input devices") + } + + init() {} + init(parsed: ParsedValues) {} + + mutating func run() async throws { + let session = AVCaptureDevice.DiscoverySession( + deviceTypes: [.microphone, .external], + mediaType: .audio, + position: .unspecified) + let devices = session.devices + if devices.isEmpty { print("no audio inputs found"); return } + for (idx, device) in devices.enumerated() { + print("[\(idx)] \(device.localizedName)") + } + } +} + +@MainActor +struct MicSet: ParsableCommand { + @Argument(help: "Device index from list") var index: Int = 0 + @Option(name: .long("config"), help: "Path to config JSON") var configPath: String? + + static var commandDescription: CommandDescription { + CommandDescription(commandName: "set", abstract: "Set default input device index") + } + + init() {} + init(parsed: ParsedValues) { + self.init() + if let value = parsed.positional.first, let intVal = Int(value) { index = intVal } + if let cfg = parsed.options["config"]?.last { configPath = cfg } + } + + mutating func run() async throws { + var cfg = try ConfigLoader.load(at: configURL) + cfg.audio.deviceIndex = index + try ConfigLoader.save(cfg, at: configURL) + print("saved device index \(index)") + } + + private var configURL: URL? { configPath.map { URL(fileURLWithPath: $0) } } +} diff --git a/Swabble/Sources/swabble/Commands/ServeCommand.swift b/Swabble/Sources/swabble/Commands/ServeCommand.swift new file mode 100644 index 0000000000000000000000000000000000000000..705ecf41a65df56762ac99c179b2fdbb60fdc5b8 --- /dev/null +++ b/Swabble/Sources/swabble/Commands/ServeCommand.swift @@ -0,0 +1,81 @@ +import Commander +import Foundation +import Swabble +import SwabbleKit + +@available(macOS 26.0, *) +@MainActor +struct ServeCommand: ParsableCommand { + @Option(name: .long("config"), help: "Path to config JSON") var configPath: String? + @Flag(name: .long("no-wake"), help: "Disable wake word") var noWake: Bool = false + + static var commandDescription: CommandDescription { + CommandDescription( + commandName: "serve", + abstract: "Run swabble in the foreground") + } + + init() {} + + init(parsed: ParsedValues) { + self.init() + if parsed.flags.contains("noWake") { noWake = true } + if let cfg = parsed.options["config"]?.last { configPath = cfg } + } + + mutating func run() async throws { + var cfg: SwabbleConfig + do { + cfg = try ConfigLoader.load(at: configURL) + } catch { + cfg = SwabbleConfig() + try ConfigLoader.save(cfg, at: configURL) + } + if noWake { + cfg.wake.enabled = false + } + + let logger = Logger(level: LogLevel(configValue: cfg.logging.level) ?? .info) + logger.info("swabble serve starting (wake: \(cfg.wake.enabled ? cfg.wake.word : "disabled"))") + let pipeline = SpeechPipeline() + do { + let stream = try await pipeline.start( + localeIdentifier: cfg.speech.localeIdentifier, + etiquette: cfg.speech.etiquetteReplacements) + for await seg in stream { + if cfg.wake.enabled { + guard Self.matchesWake(text: seg.text, cfg: cfg) else { continue } + } + let stripped = Self.stripWake(text: seg.text, cfg: cfg) + let job = HookJob(text: stripped, timestamp: Date()) + let executor = HookExecutor(config: cfg) + try await executor.run(job: job) + if cfg.transcripts.enabled { + await TranscriptsStore.shared.append(text: stripped) + } + if seg.isFinal { + logger.info("final: \(stripped)") + } else { + logger.debug("partial: \(stripped)") + } + } + } catch { + logger.error("serve error: \(error)") + throw error + } + } + + private var configURL: URL? { + configPath.map { URL(fileURLWithPath: $0) } + } + + private static func matchesWake(text: String, cfg: SwabbleConfig) -> Bool { + let triggers = [cfg.wake.word] + cfg.wake.aliases + return WakeWordGate.matchesTextOnly(text: text, triggers: triggers) + } + + private static func stripWake(text: String, cfg: SwabbleConfig) -> String { + let triggers = [cfg.wake.word] + cfg.wake.aliases + return WakeWordGate.stripWake(text: text, triggers: triggers) + } +} diff --git a/Swabble/Sources/swabble/Commands/ServiceCommands.swift b/Swabble/Sources/swabble/Commands/ServiceCommands.swift new file mode 100644 index 0000000000000000000000000000000000000000..8690e95628d4e387d6c297cd7b92173fcae52379 --- /dev/null +++ b/Swabble/Sources/swabble/Commands/ServiceCommands.swift @@ -0,0 +1,77 @@ +import Commander +import Foundation + +@MainActor +struct ServiceRootCommand: ParsableCommand { + static var commandDescription: CommandDescription { + CommandDescription( + commandName: "service", + abstract: "Manage launchd agent", + subcommands: [ServiceInstall.self, ServiceUninstall.self, ServiceStatus.self]) + } +} + +private enum LaunchdHelper { + static let label = "com.swabble.agent" + + static var plistURL: URL { + FileManager.default + .homeDirectoryForCurrentUser + .appendingPathComponent("Library/LaunchAgents/\(label).plist") + } + + static func writePlist(executable: String) throws { + let plist: [String: Any] = [ + "Label": label, + "ProgramArguments": [executable, "serve"], + "RunAtLoad": true, + "KeepAlive": true + ] + let data = try PropertyListSerialization.data(fromPropertyList: plist, format: .xml, options: 0) + try data.write(to: plistURL) + } + + static func removePlist() throws { + try? FileManager.default.removeItem(at: plistURL) + } +} + +@MainActor +struct ServiceInstall: ParsableCommand { + static var commandDescription: CommandDescription { + CommandDescription(commandName: "install", abstract: "Install user launch agent") + } + + mutating func run() async throws { + let exe = CommandLine.arguments.first ?? "/usr/local/bin/swabble" + try LaunchdHelper.writePlist(executable: exe) + print("launchctl load -w \(LaunchdHelper.plistURL.path)") + } +} + +@MainActor +struct ServiceUninstall: ParsableCommand { + static var commandDescription: CommandDescription { + CommandDescription(commandName: "uninstall", abstract: "Remove launch agent") + } + + mutating func run() async throws { + try LaunchdHelper.removePlist() + print("launchctl bootout gui/$(id -u)/\(LaunchdHelper.label)") + } +} + +@MainActor +struct ServiceStatus: ParsableCommand { + static var commandDescription: CommandDescription { + CommandDescription(commandName: "status", abstract: "Show launch agent status") + } + + mutating func run() async throws { + if FileManager.default.fileExists(atPath: LaunchdHelper.plistURL.path) { + print("plist present at \(LaunchdHelper.plistURL.path)") + } else { + print("launchd plist not installed") + } + } +} diff --git a/Swabble/Sources/swabble/Commands/SetupCommand.swift b/Swabble/Sources/swabble/Commands/SetupCommand.swift new file mode 100644 index 0000000000000000000000000000000000000000..469de233d1103952ebfa06a90e56d27fbb1fc48e --- /dev/null +++ b/Swabble/Sources/swabble/Commands/SetupCommand.swift @@ -0,0 +1,26 @@ +import Commander +import Foundation +import Swabble + +@MainActor +struct SetupCommand: ParsableCommand { + static var commandDescription: CommandDescription { + CommandDescription(commandName: "setup", abstract: "Write default config") + } + + @Option(name: .long("config"), help: "Path to config JSON") var configPath: String? + + init() {} + init(parsed: ParsedValues) { + self.init() + if let cfg = parsed.options["config"]?.last { configPath = cfg } + } + + mutating func run() async throws { + let cfg = SwabbleConfig() + try ConfigLoader.save(cfg, at: configURL) + print("wrote config to \(configURL?.path ?? SwabbleConfig.defaultPath.path)") + } + + private var configURL: URL? { configPath.map { URL(fileURLWithPath: $0) } } +} diff --git a/Swabble/Sources/swabble/Commands/StartStopCommands.swift b/Swabble/Sources/swabble/Commands/StartStopCommands.swift new file mode 100644 index 0000000000000000000000000000000000000000..641cd923a0d423a66c45ef04f962d2fa98fa1910 --- /dev/null +++ b/Swabble/Sources/swabble/Commands/StartStopCommands.swift @@ -0,0 +1,35 @@ +import Commander +import Foundation + +@MainActor +struct StartCommand: ParsableCommand { + static var commandDescription: CommandDescription { + CommandDescription(commandName: "start", abstract: "Start swabble (foreground placeholder)") + } + + mutating func run() async throws { + print("start: launchd helper not implemented; run 'swabble serve' instead") + } +} + +@MainActor +struct StopCommand: ParsableCommand { + static var commandDescription: CommandDescription { + CommandDescription(commandName: "stop", abstract: "Stop swabble (placeholder)") + } + + mutating func run() async throws { + print("stop: launchd helper not implemented yet") + } +} + +@MainActor +struct RestartCommand: ParsableCommand { + static var commandDescription: CommandDescription { + CommandDescription(commandName: "restart", abstract: "Restart swabble (placeholder)") + } + + mutating func run() async throws { + print("restart: launchd helper not implemented yet") + } +} diff --git a/Swabble/Sources/swabble/Commands/StatusCommand.swift b/Swabble/Sources/swabble/Commands/StatusCommand.swift new file mode 100644 index 0000000000000000000000000000000000000000..19db16117ab00f7bfd197ff3268cbc3cb5a35bbc --- /dev/null +++ b/Swabble/Sources/swabble/Commands/StatusCommand.swift @@ -0,0 +1,34 @@ +import Commander +import Foundation +import Swabble + +@MainActor +struct StatusCommand: ParsableCommand { + static var commandDescription: CommandDescription { + CommandDescription(commandName: "status", abstract: "Show daemon state") + } + + @Option(name: .long("config"), help: "Path to config JSON") var configPath: String? + + init() {} + init(parsed: ParsedValues) { + self.init() + if let cfg = parsed.options["config"]?.last { configPath = cfg } + } + + mutating func run() async throws { + let cfg = try? ConfigLoader.load(at: configURL) + let wake = cfg?.wake.word ?? "clawd" + let wakeEnabled = cfg?.wake.enabled ?? false + let latest = await TranscriptsStore.shared.latest().suffix(3) + print("wake: \(wakeEnabled ? wake : "disabled")") + if latest.isEmpty { + print("transcripts: (none yet)") + } else { + print("last transcripts:") + latest.forEach { print("- \($0)") } + } + } + + private var configURL: URL? { configPath.map { URL(fileURLWithPath: $0) } } +} diff --git a/Swabble/Sources/swabble/Commands/TailLogCommand.swift b/Swabble/Sources/swabble/Commands/TailLogCommand.swift new file mode 100644 index 0000000000000000000000000000000000000000..451ed37de41644f76830f439a5c5d76b1d08f1ae --- /dev/null +++ b/Swabble/Sources/swabble/Commands/TailLogCommand.swift @@ -0,0 +1,20 @@ +import Commander +import Foundation +import Swabble + +@MainActor +struct TailLogCommand: ParsableCommand { + static var commandDescription: CommandDescription { + CommandDescription(commandName: "tail-log", abstract: "Tail recent transcripts") + } + + init() {} + init(parsed: ParsedValues) {} + + mutating func run() async throws { + let latest = await TranscriptsStore.shared.latest() + for line in latest.suffix(10) { + print(line) + } + } +} diff --git a/Swabble/Sources/swabble/Commands/TestHookCommand.swift b/Swabble/Sources/swabble/Commands/TestHookCommand.swift new file mode 100644 index 0000000000000000000000000000000000000000..226776ceb89d9b6b9b371aa1f44c6e6b7805f046 --- /dev/null +++ b/Swabble/Sources/swabble/Commands/TestHookCommand.swift @@ -0,0 +1,30 @@ +import Commander +import Foundation +import Swabble + +@MainActor +struct TestHookCommand: ParsableCommand { + @Argument(help: "Text to send to hook") var text: String + @Option(name: .long("config"), help: "Path to config JSON") var configPath: String? + + static var commandDescription: CommandDescription { + CommandDescription(commandName: "test-hook", abstract: "Invoke the configured hook with text") + } + + init() {} + + init(parsed: ParsedValues) { + self.init() + if let positional = parsed.positional.first { text = positional } + if let cfg = parsed.options["config"]?.last { configPath = cfg } + } + + mutating func run() async throws { + let cfg = try ConfigLoader.load(at: configURL) + let executor = HookExecutor(config: cfg) + try await executor.run(job: HookJob(text: text, timestamp: Date())) + print("hook invoked") + } + + private var configURL: URL? { configPath.map { URL(fileURLWithPath: $0) } } +} diff --git a/Swabble/Sources/swabble/Commands/TranscribeCommand.swift b/Swabble/Sources/swabble/Commands/TranscribeCommand.swift new file mode 100644 index 0000000000000000000000000000000000000000..1bedca3fc0adc98612da2706639c0fbab4a5d5c9 --- /dev/null +++ b/Swabble/Sources/swabble/Commands/TranscribeCommand.swift @@ -0,0 +1,61 @@ +import AVFoundation +import Commander +import Foundation +import Speech +import Swabble + +@MainActor +struct TranscribeCommand: ParsableCommand { + @Argument(help: "Path to audio/video file") var inputFile: String = "" + @Option(name: .long("locale"), help: "Locale identifier", parsing: .singleValue) var locale: String = Locale.current + .identifier + @Flag(help: "Censor etiquette-sensitive content") var censor: Bool = false + @Option(name: .long("output"), help: "Output file path") var outputFile: String? + @Option(name: .long("format"), help: "Output format txt|srt") var format: String = "txt" + @Option(name: .long("max-length"), help: "Max sentence length for srt") var maxLength: Int = 40 + + static var commandDescription: CommandDescription { + CommandDescription( + commandName: "transcribe", + abstract: "Transcribe a media file locally") + } + + init() {} + + init(parsed: ParsedValues) { + self.init() + if let positional = parsed.positional.first { inputFile = positional } + if let loc = parsed.options["locale"]?.last { locale = loc } + if parsed.flags.contains("censor") { censor = true } + if let out = parsed.options["output"]?.last { outputFile = out } + if let fmt = parsed.options["format"]?.last { format = fmt } + if let len = parsed.options["maxLength"]?.last, let intVal = Int(len) { maxLength = intVal } + } + + mutating func run() async throws { + let fileURL = URL(fileURLWithPath: inputFile) + let audioFile = try AVAudioFile(forReading: fileURL) + + let outputFormat = OutputFormat(rawValue: format) ?? .txt + + let transcriber = SpeechTranscriber( + locale: Locale(identifier: locale), + transcriptionOptions: censor ? [.etiquetteReplacements] : [], + reportingOptions: [], + attributeOptions: outputFormat.needsAudioTimeRange ? [.audioTimeRange] : []) + let analyzer = SpeechAnalyzer(modules: [transcriber]) + try await analyzer.start(inputAudioFile: audioFile, finishAfterFile: true) + + var transcript: AttributedString = "" + for try await result in transcriber.results { + transcript += result.text + } + + let output = outputFormat.text(for: transcript, maxLength: maxLength) + if let path = outputFile { + try output.write(to: URL(fileURLWithPath: path), atomically: false, encoding: .utf8) + } else { + print(output) + } + } +} diff --git a/Swabble/Sources/swabble/main.swift b/Swabble/Sources/swabble/main.swift new file mode 100644 index 0000000000000000000000000000000000000000..a534c68d969c53967c83286bc5282594e1dc0bae --- /dev/null +++ b/Swabble/Sources/swabble/main.swift @@ -0,0 +1,151 @@ +import Commander +import Foundation + +@available(macOS 26.0, *) +@MainActor +private func runCLI() async -> Int32 { + do { + let descriptors = CLIRegistry.descriptors + let program = Program(descriptors: descriptors) + let invocation = try program.resolve(argv: CommandLine.arguments) + try await dispatch(invocation: invocation) + return 0 + } catch { + fputs("error: \(error)\n", stderr) + return 1 + } +} + +@available(macOS 26.0, *) +@MainActor +private func dispatch(invocation: CommandInvocation) async throws { + let parsed = invocation.parsedValues + let path = invocation.path + guard let first = path.first else { throw CommanderProgramError.missingCommand } + + switch first { + case "swabble": + try await dispatchSwabble(parsed: parsed, path: path) + default: + throw CommanderProgramError.unknownCommand(first) + } +} + +@available(macOS 26.0, *) +@MainActor +private func dispatchSwabble(parsed: ParsedValues, path: [String]) async throws { + let sub = try subcommand(path, index: 1, command: "swabble") + switch sub { + case "mic": + try await dispatchMic(parsed: parsed, path: path) + case "service": + try await dispatchService(path: path) + default: + let handlers = swabbleHandlers(parsed: parsed) + guard let handler = handlers[sub] else { + throw CommanderProgramError.unknownSubcommand(command: "swabble", name: sub) + } + try await handler() + } +} + +@available(macOS 26.0, *) +@MainActor +private func swabbleHandlers(parsed: ParsedValues) -> [String: () async throws -> Void] { + [ + "serve": { + var cmd = ServeCommand(parsed: parsed) + try await cmd.run() + }, + "transcribe": { + var cmd = TranscribeCommand(parsed: parsed) + try await cmd.run() + }, + "test-hook": { + var cmd = TestHookCommand(parsed: parsed) + try await cmd.run() + }, + "doctor": { + var cmd = DoctorCommand(parsed: parsed) + try await cmd.run() + }, + "setup": { + var cmd = SetupCommand(parsed: parsed) + try await cmd.run() + }, + "health": { + var cmd = HealthCommand(parsed: parsed) + try await cmd.run() + }, + "tail-log": { + var cmd = TailLogCommand(parsed: parsed) + try await cmd.run() + }, + "start": { + var cmd = StartCommand() + try await cmd.run() + }, + "stop": { + var cmd = StopCommand() + try await cmd.run() + }, + "restart": { + var cmd = RestartCommand() + try await cmd.run() + }, + "status": { + var cmd = StatusCommand() + try await cmd.run() + } + ] +} + +@available(macOS 26.0, *) +@MainActor +private func dispatchMic(parsed: ParsedValues, path: [String]) async throws { + let micSub = try subcommand(path, index: 2, command: "mic") + switch micSub { + case "list": + var cmd = MicList(parsed: parsed) + try await cmd.run() + case "set": + var cmd = MicSet(parsed: parsed) + try await cmd.run() + default: + throw CommanderProgramError.unknownSubcommand(command: "mic", name: micSub) + } +} + +@available(macOS 26.0, *) +@MainActor +private func dispatchService(path: [String]) async throws { + let svcSub = try subcommand(path, index: 2, command: "service") + switch svcSub { + case "install": + var cmd = ServiceInstall() + try await cmd.run() + case "uninstall": + var cmd = ServiceUninstall() + try await cmd.run() + case "status": + var cmd = ServiceStatus() + try await cmd.run() + default: + throw CommanderProgramError.unknownSubcommand(command: "service", name: svcSub) + } +} + +private func subcommand(_ path: [String], index: Int, command: String) throws -> String { + guard path.count > index else { + throw CommanderProgramError.missingSubcommand(command: command) + } + return path[index] +} + +if #available(macOS 26.0, *) { + let exitCode = await runCLI() + exit(exitCode) +} else { + fputs("error: swabble requires macOS 26 or newer\n", stderr) + exit(1) +} diff --git a/Swabble/Tests/SwabbleKitTests/WakeWordGateTests.swift b/Swabble/Tests/SwabbleKitTests/WakeWordGateTests.swift new file mode 100644 index 0000000000000000000000000000000000000000..5cc283c35aea2307efb3374b8565bbddbbc56092 --- /dev/null +++ b/Swabble/Tests/SwabbleKitTests/WakeWordGateTests.swift @@ -0,0 +1,63 @@ +import Foundation +import SwabbleKit +import Testing + +@Suite struct WakeWordGateTests { + @Test func matchRequiresGapAfterTrigger() { + let transcript = "hey clawd do thing" + let segments = makeSegments( + transcript: transcript, + words: [ + ("hey", 0.0, 0.1), + ("clawd", 0.2, 0.1), + ("do", 0.35, 0.1), + ("thing", 0.5, 0.1), + ]) + let config = WakeWordGateConfig(triggers: ["clawd"], minPostTriggerGap: 0.3) + #expect(WakeWordGate.match(transcript: transcript, segments: segments, config: config) == nil) + } + + @Test func matchAllowsGapAndExtractsCommand() { + let transcript = "hey clawd do thing" + let segments = makeSegments( + transcript: transcript, + words: [ + ("hey", 0.0, 0.1), + ("clawd", 0.2, 0.1), + ("do", 0.9, 0.1), + ("thing", 1.1, 0.1), + ]) + let config = WakeWordGateConfig(triggers: ["clawd"], minPostTriggerGap: 0.3) + let match = WakeWordGate.match(transcript: transcript, segments: segments, config: config) + #expect(match?.command == "do thing") + } + + @Test func matchHandlesMultiWordTriggers() { + let transcript = "hey clawd do it" + let segments = makeSegments( + transcript: transcript, + words: [ + ("hey", 0.0, 0.1), + ("clawd", 0.2, 0.1), + ("do", 0.8, 0.1), + ("it", 1.0, 0.1), + ]) + let config = WakeWordGateConfig(triggers: ["hey clawd"], minPostTriggerGap: 0.3) + let match = WakeWordGate.match(transcript: transcript, segments: segments, config: config) + #expect(match?.command == "do it") + } +} + +private func makeSegments( + transcript: String, + words: [(String, TimeInterval, TimeInterval)]) +-> [WakeWordSegment] { + var searchStart = transcript.startIndex + var output: [WakeWordSegment] = [] + for (word, start, duration) in words { + let range = transcript.range(of: word, range: searchStart../dev/null; then + echo "swiftlint not installed" >&2 + exit 1 +fi +swiftlint --config "$CONFIG" diff --git a/apps/android/.gitignore b/apps/android/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..68bfc099e369fe52ed13efffd81fa9353e314891 --- /dev/null +++ b/apps/android/.gitignore @@ -0,0 +1,5 @@ +.gradle/ +**/build/ +local.properties +.idea/ +**/*.iml diff --git a/apps/android/README.md b/apps/android/README.md new file mode 100644 index 0000000000000000000000000000000000000000..c2ae5a2179bf1f383720094527de08c415dcaaed --- /dev/null +++ b/apps/android/README.md @@ -0,0 +1,51 @@ +## OpenClaw Node (Android) (internal) + +Modern Android node app: connects to the **Gateway WebSocket** (`_openclaw-gw._tcp`) and exposes **Canvas + Chat + Camera**. + +Notes: +- The node keeps the connection alive via a **foreground service** (persistent notification with a Disconnect action). +- Chat always uses the shared session key **`main`** (same session across iOS/macOS/WebChat/Android). +- Supports modern Android only (`minSdk 31`, Kotlin + Jetpack Compose). + +## Open in Android Studio +- Open the folder `apps/android`. + +## Build / Run + +```bash +cd apps/android +./gradlew :app:assembleDebug +./gradlew :app:installDebug +./gradlew :app:testDebugUnitTest +``` + +`gradlew` auto-detects the Android SDK at `~/Library/Android/sdk` (macOS default) if `ANDROID_SDK_ROOT` / `ANDROID_HOME` are unset. + +## Connect / Pair + +1) Start the gateway (on your “master” machine): +```bash +pnpm openclaw gateway --port 18789 --verbose +``` + +2) In the Android app: +- Open **Settings** +- Either select a discovered gateway under **Discovered Gateways**, or use **Advanced → Manual Gateway** (host + port). + +3) Approve pairing (on the gateway machine): +```bash +openclaw nodes pending +openclaw nodes approve +``` + +More details: `docs/platforms/android.md`. + +## Permissions + +- Discovery: + - Android 13+ (`API 33+`): `NEARBY_WIFI_DEVICES` + - Android 12 and below: `ACCESS_FINE_LOCATION` (required for NSD scanning) +- Foreground service notification (Android 13+): `POST_NOTIFICATIONS` +- Camera: + - `CAMERA` for `camera.snap` and `camera.clip` + - `RECORD_AUDIO` for `camera.clip` when `includeAudio=true` diff --git a/apps/android/app/build.gradle.kts b/apps/android/app/build.gradle.kts new file mode 100644 index 0000000000000000000000000000000000000000..8279b6bd97004e06e33fd3241872630b207ca807 --- /dev/null +++ b/apps/android/app/build.gradle.kts @@ -0,0 +1,128 @@ +import com.android.build.api.variant.impl.VariantOutputImpl + +plugins { + id("com.android.application") + id("org.jetbrains.kotlin.android") + id("org.jetbrains.kotlin.plugin.compose") + id("org.jetbrains.kotlin.plugin.serialization") +} + +android { + namespace = "ai.openclaw.android" + compileSdk = 36 + + sourceSets { + getByName("main") { + assets.srcDir(file("../../shared/OpenClawKit/Sources/OpenClawKit/Resources")) + } + } + + defaultConfig { + applicationId = "ai.openclaw.android" + minSdk = 31 + targetSdk = 36 + versionCode = 202601290 + versionName = "2026.1.30" + } + + buildTypes { + release { + isMinifyEnabled = false + } + } + + buildFeatures { + compose = true + buildConfig = true + } + + compileOptions { + sourceCompatibility = JavaVersion.VERSION_17 + targetCompatibility = JavaVersion.VERSION_17 + } + + packaging { + resources { + excludes += "/META-INF/{AL2.0,LGPL2.1}" + } + } + + lint { + disable += setOf("IconLauncherShape") + warningsAsErrors = true + } + + testOptions { + unitTests.isIncludeAndroidResources = true + } +} + +androidComponents { + onVariants { variant -> + variant.outputs + .filterIsInstance() + .forEach { output -> + val versionName = output.versionName.orNull ?: "0" + val buildType = variant.buildType + + val outputFileName = "openclaw-${versionName}-${buildType}.apk" + output.outputFileName = outputFileName + } + } +} +kotlin { + compilerOptions { + jvmTarget.set(org.jetbrains.kotlin.gradle.dsl.JvmTarget.JVM_17) + allWarningsAsErrors.set(true) + } +} + +dependencies { + val composeBom = platform("androidx.compose:compose-bom:2025.12.00") + implementation(composeBom) + androidTestImplementation(composeBom) + + implementation("androidx.core:core-ktx:1.17.0") + implementation("androidx.lifecycle:lifecycle-runtime-ktx:2.10.0") + implementation("androidx.activity:activity-compose:1.12.2") + implementation("androidx.webkit:webkit:1.15.0") + + implementation("androidx.compose.ui:ui") + implementation("androidx.compose.ui:ui-tooling-preview") + implementation("androidx.compose.material3:material3") + implementation("androidx.compose.material:material-icons-extended") + implementation("androidx.navigation:navigation-compose:2.9.6") + + debugImplementation("androidx.compose.ui:ui-tooling") + + // Material Components (XML theme + resources) + implementation("com.google.android.material:material:1.13.0") + + implementation("org.jetbrains.kotlinx:kotlinx-coroutines-android:1.10.2") + implementation("org.jetbrains.kotlinx:kotlinx-serialization-json:1.9.0") + + implementation("androidx.security:security-crypto:1.1.0") + implementation("androidx.exifinterface:exifinterface:1.4.2") + implementation("com.squareup.okhttp3:okhttp:5.3.2") + + // CameraX (for node.invoke camera.* parity) + implementation("androidx.camera:camera-core:1.5.2") + implementation("androidx.camera:camera-camera2:1.5.2") + implementation("androidx.camera:camera-lifecycle:1.5.2") + implementation("androidx.camera:camera-video:1.5.2") + implementation("androidx.camera:camera-view:1.5.2") + + // Unicast DNS-SD (Wide-Area Bonjour) for tailnet discovery domains. + implementation("dnsjava:dnsjava:3.6.4") + + testImplementation("junit:junit:4.13.2") + testImplementation("org.jetbrains.kotlinx:kotlinx-coroutines-test:1.10.2") + testImplementation("io.kotest:kotest-runner-junit5-jvm:6.0.7") + testImplementation("io.kotest:kotest-assertions-core-jvm:6.0.7") + testImplementation("org.robolectric:robolectric:4.16") + testRuntimeOnly("org.junit.vintage:junit-vintage-engine:6.0.2") +} + +tasks.withType().configureEach { + useJUnitPlatform() +} diff --git a/apps/android/app/src/main/AndroidManifest.xml b/apps/android/app/src/main/AndroidManifest.xml new file mode 100644 index 0000000000000000000000000000000000000000..bc0de1f87c46eb7c61bdc925a798f3b18ee06b77 --- /dev/null +++ b/apps/android/app/src/main/AndroidManifest.xml @@ -0,0 +1,49 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/apps/android/app/src/main/java/ai/openclaw/android/CameraHudState.kt b/apps/android/app/src/main/java/ai/openclaw/android/CameraHudState.kt new file mode 100644 index 0000000000000000000000000000000000000000..636c31bdd3c3218a8abe3a0d831f5cc04ea6e4df --- /dev/null +++ b/apps/android/app/src/main/java/ai/openclaw/android/CameraHudState.kt @@ -0,0 +1,14 @@ +package ai.openclaw.android + +enum class CameraHudKind { + Photo, + Recording, + Success, + Error, +} + +data class CameraHudState( + val token: Long, + val kind: CameraHudKind, + val message: String, +) diff --git a/apps/android/app/src/main/java/ai/openclaw/android/DeviceNames.kt b/apps/android/app/src/main/java/ai/openclaw/android/DeviceNames.kt new file mode 100644 index 0000000000000000000000000000000000000000..3c44a3bb4f7934a17dfe15ce3277adc0c1fb8749 --- /dev/null +++ b/apps/android/app/src/main/java/ai/openclaw/android/DeviceNames.kt @@ -0,0 +1,26 @@ +package ai.openclaw.android + +import android.content.Context +import android.os.Build +import android.provider.Settings + +object DeviceNames { + fun bestDefaultNodeName(context: Context): String { + val deviceName = + runCatching { + Settings.Global.getString(context.contentResolver, "device_name") + } + .getOrNull() + ?.trim() + .orEmpty() + + if (deviceName.isNotEmpty()) return deviceName + + val model = + listOfNotNull(Build.MANUFACTURER?.takeIf { it.isNotBlank() }, Build.MODEL?.takeIf { it.isNotBlank() }) + .joinToString(" ") + .trim() + + return model.ifEmpty { "Android Node" } + } +} diff --git a/apps/android/app/src/main/java/ai/openclaw/android/LocationMode.kt b/apps/android/app/src/main/java/ai/openclaw/android/LocationMode.kt new file mode 100644 index 0000000000000000000000000000000000000000..eb9c84428e04e7a36b485b6532710202b5a8ca60 --- /dev/null +++ b/apps/android/app/src/main/java/ai/openclaw/android/LocationMode.kt @@ -0,0 +1,15 @@ +package ai.openclaw.android + +enum class LocationMode(val rawValue: String) { + Off("off"), + WhileUsing("whileUsing"), + Always("always"), + ; + + companion object { + fun fromRawValue(raw: String?): LocationMode { + val normalized = raw?.trim()?.lowercase() + return entries.firstOrNull { it.rawValue.lowercase() == normalized } ?: Off + } + } +} diff --git a/apps/android/app/src/main/java/ai/openclaw/android/MainActivity.kt b/apps/android/app/src/main/java/ai/openclaw/android/MainActivity.kt new file mode 100644 index 0000000000000000000000000000000000000000..2bbfd8712f92d951e4c093b4cae7ece1b5868e17 --- /dev/null +++ b/apps/android/app/src/main/java/ai/openclaw/android/MainActivity.kt @@ -0,0 +1,130 @@ +package ai.openclaw.android + +import android.Manifest +import android.content.pm.ApplicationInfo +import android.os.Bundle +import android.os.Build +import android.view.WindowManager +import android.webkit.WebView +import androidx.activity.ComponentActivity +import androidx.activity.compose.setContent +import androidx.activity.viewModels +import androidx.compose.material3.Surface +import androidx.compose.ui.Modifier +import androidx.core.content.ContextCompat +import androidx.core.view.WindowCompat +import androidx.core.view.WindowInsetsCompat +import androidx.core.view.WindowInsetsControllerCompat +import androidx.lifecycle.Lifecycle +import androidx.lifecycle.lifecycleScope +import androidx.lifecycle.repeatOnLifecycle +import ai.openclaw.android.ui.RootScreen +import ai.openclaw.android.ui.OpenClawTheme +import kotlinx.coroutines.launch + +class MainActivity : ComponentActivity() { + private val viewModel: MainViewModel by viewModels() + private lateinit var permissionRequester: PermissionRequester + private lateinit var screenCaptureRequester: ScreenCaptureRequester + + override fun onCreate(savedInstanceState: Bundle?) { + super.onCreate(savedInstanceState) + val isDebuggable = (applicationInfo.flags and ApplicationInfo.FLAG_DEBUGGABLE) != 0 + WebView.setWebContentsDebuggingEnabled(isDebuggable) + applyImmersiveMode() + requestDiscoveryPermissionsIfNeeded() + requestNotificationPermissionIfNeeded() + NodeForegroundService.start(this) + permissionRequester = PermissionRequester(this) + screenCaptureRequester = ScreenCaptureRequester(this) + viewModel.camera.attachLifecycleOwner(this) + viewModel.camera.attachPermissionRequester(permissionRequester) + viewModel.sms.attachPermissionRequester(permissionRequester) + viewModel.screenRecorder.attachScreenCaptureRequester(screenCaptureRequester) + viewModel.screenRecorder.attachPermissionRequester(permissionRequester) + + lifecycleScope.launch { + repeatOnLifecycle(Lifecycle.State.STARTED) { + viewModel.preventSleep.collect { enabled -> + if (enabled) { + window.addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON) + } else { + window.clearFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON) + } + } + } + } + + setContent { + OpenClawTheme { + Surface(modifier = Modifier) { + RootScreen(viewModel = viewModel) + } + } + } + } + + override fun onResume() { + super.onResume() + applyImmersiveMode() + } + + override fun onWindowFocusChanged(hasFocus: Boolean) { + super.onWindowFocusChanged(hasFocus) + if (hasFocus) { + applyImmersiveMode() + } + } + + override fun onStart() { + super.onStart() + viewModel.setForeground(true) + } + + override fun onStop() { + viewModel.setForeground(false) + super.onStop() + } + + private fun applyImmersiveMode() { + WindowCompat.setDecorFitsSystemWindows(window, false) + val controller = WindowInsetsControllerCompat(window, window.decorView) + controller.systemBarsBehavior = + WindowInsetsControllerCompat.BEHAVIOR_SHOW_TRANSIENT_BARS_BY_SWIPE + controller.hide(WindowInsetsCompat.Type.systemBars()) + } + + private fun requestDiscoveryPermissionsIfNeeded() { + if (Build.VERSION.SDK_INT >= 33) { + val ok = + ContextCompat.checkSelfPermission( + this, + Manifest.permission.NEARBY_WIFI_DEVICES, + ) == android.content.pm.PackageManager.PERMISSION_GRANTED + if (!ok) { + requestPermissions(arrayOf(Manifest.permission.NEARBY_WIFI_DEVICES), 100) + } + } else { + val ok = + ContextCompat.checkSelfPermission( + this, + Manifest.permission.ACCESS_FINE_LOCATION, + ) == android.content.pm.PackageManager.PERMISSION_GRANTED + if (!ok) { + requestPermissions(arrayOf(Manifest.permission.ACCESS_FINE_LOCATION), 101) + } + } + } + + private fun requestNotificationPermissionIfNeeded() { + if (Build.VERSION.SDK_INT < 33) return + val ok = + ContextCompat.checkSelfPermission( + this, + Manifest.permission.POST_NOTIFICATIONS, + ) == android.content.pm.PackageManager.PERMISSION_GRANTED + if (!ok) { + requestPermissions(arrayOf(Manifest.permission.POST_NOTIFICATIONS), 102) + } + } +} diff --git a/apps/android/app/src/main/java/ai/openclaw/android/MainViewModel.kt b/apps/android/app/src/main/java/ai/openclaw/android/MainViewModel.kt new file mode 100644 index 0000000000000000000000000000000000000000..0868fcb796ffdd40912cd4fa28e23b11e6591eed --- /dev/null +++ b/apps/android/app/src/main/java/ai/openclaw/android/MainViewModel.kt @@ -0,0 +1,174 @@ +package ai.openclaw.android + +import android.app.Application +import androidx.lifecycle.AndroidViewModel +import ai.openclaw.android.gateway.GatewayEndpoint +import ai.openclaw.android.chat.OutgoingAttachment +import ai.openclaw.android.node.CameraCaptureManager +import ai.openclaw.android.node.CanvasController +import ai.openclaw.android.node.ScreenRecordManager +import ai.openclaw.android.node.SmsManager +import kotlinx.coroutines.flow.StateFlow + +class MainViewModel(app: Application) : AndroidViewModel(app) { + private val runtime: NodeRuntime = (app as NodeApp).runtime + + val canvas: CanvasController = runtime.canvas + val camera: CameraCaptureManager = runtime.camera + val screenRecorder: ScreenRecordManager = runtime.screenRecorder + val sms: SmsManager = runtime.sms + + val gateways: StateFlow> = runtime.gateways + val discoveryStatusText: StateFlow = runtime.discoveryStatusText + + val isConnected: StateFlow = runtime.isConnected + val statusText: StateFlow = runtime.statusText + val serverName: StateFlow = runtime.serverName + val remoteAddress: StateFlow = runtime.remoteAddress + val isForeground: StateFlow = runtime.isForeground + val seamColorArgb: StateFlow = runtime.seamColorArgb + val mainSessionKey: StateFlow = runtime.mainSessionKey + + val cameraHud: StateFlow = runtime.cameraHud + val cameraFlashToken: StateFlow = runtime.cameraFlashToken + val screenRecordActive: StateFlow = runtime.screenRecordActive + + val instanceId: StateFlow = runtime.instanceId + val displayName: StateFlow = runtime.displayName + val cameraEnabled: StateFlow = runtime.cameraEnabled + val locationMode: StateFlow = runtime.locationMode + val locationPreciseEnabled: StateFlow = runtime.locationPreciseEnabled + val preventSleep: StateFlow = runtime.preventSleep + val wakeWords: StateFlow> = runtime.wakeWords + val voiceWakeMode: StateFlow = runtime.voiceWakeMode + val voiceWakeStatusText: StateFlow = runtime.voiceWakeStatusText + val voiceWakeIsListening: StateFlow = runtime.voiceWakeIsListening + val talkEnabled: StateFlow = runtime.talkEnabled + val talkStatusText: StateFlow = runtime.talkStatusText + val talkIsListening: StateFlow = runtime.talkIsListening + val talkIsSpeaking: StateFlow = runtime.talkIsSpeaking + val manualEnabled: StateFlow = runtime.manualEnabled + val manualHost: StateFlow = runtime.manualHost + val manualPort: StateFlow = runtime.manualPort + val manualTls: StateFlow = runtime.manualTls + val canvasDebugStatusEnabled: StateFlow = runtime.canvasDebugStatusEnabled + + val chatSessionKey: StateFlow = runtime.chatSessionKey + val chatSessionId: StateFlow = runtime.chatSessionId + val chatMessages = runtime.chatMessages + val chatError: StateFlow = runtime.chatError + val chatHealthOk: StateFlow = runtime.chatHealthOk + val chatThinkingLevel: StateFlow = runtime.chatThinkingLevel + val chatStreamingAssistantText: StateFlow = runtime.chatStreamingAssistantText + val chatPendingToolCalls = runtime.chatPendingToolCalls + val chatSessions = runtime.chatSessions + val pendingRunCount: StateFlow = runtime.pendingRunCount + + fun setForeground(value: Boolean) { + runtime.setForeground(value) + } + + fun setDisplayName(value: String) { + runtime.setDisplayName(value) + } + + fun setCameraEnabled(value: Boolean) { + runtime.setCameraEnabled(value) + } + + fun setLocationMode(mode: LocationMode) { + runtime.setLocationMode(mode) + } + + fun setLocationPreciseEnabled(value: Boolean) { + runtime.setLocationPreciseEnabled(value) + } + + fun setPreventSleep(value: Boolean) { + runtime.setPreventSleep(value) + } + + fun setManualEnabled(value: Boolean) { + runtime.setManualEnabled(value) + } + + fun setManualHost(value: String) { + runtime.setManualHost(value) + } + + fun setManualPort(value: Int) { + runtime.setManualPort(value) + } + + fun setManualTls(value: Boolean) { + runtime.setManualTls(value) + } + + fun setCanvasDebugStatusEnabled(value: Boolean) { + runtime.setCanvasDebugStatusEnabled(value) + } + + fun setWakeWords(words: List) { + runtime.setWakeWords(words) + } + + fun resetWakeWordsDefaults() { + runtime.resetWakeWordsDefaults() + } + + fun setVoiceWakeMode(mode: VoiceWakeMode) { + runtime.setVoiceWakeMode(mode) + } + + fun setTalkEnabled(enabled: Boolean) { + runtime.setTalkEnabled(enabled) + } + + fun refreshGatewayConnection() { + runtime.refreshGatewayConnection() + } + + fun connect(endpoint: GatewayEndpoint) { + runtime.connect(endpoint) + } + + fun connectManual() { + runtime.connectManual() + } + + fun disconnect() { + runtime.disconnect() + } + + fun handleCanvasA2UIActionFromWebView(payloadJson: String) { + runtime.handleCanvasA2UIActionFromWebView(payloadJson) + } + + fun loadChat(sessionKey: String) { + runtime.loadChat(sessionKey) + } + + fun refreshChat() { + runtime.refreshChat() + } + + fun refreshChatSessions(limit: Int? = null) { + runtime.refreshChatSessions(limit = limit) + } + + fun setChatThinkingLevel(level: String) { + runtime.setChatThinkingLevel(level) + } + + fun switchChatSession(sessionKey: String) { + runtime.switchChatSession(sessionKey) + } + + fun abortChat() { + runtime.abortChat() + } + + fun sendChat(message: String, thinking: String, attachments: List) { + runtime.sendChat(message = message, thinking = thinking, attachments = attachments) + } +} diff --git a/apps/android/app/src/main/java/ai/openclaw/android/NodeApp.kt b/apps/android/app/src/main/java/ai/openclaw/android/NodeApp.kt new file mode 100644 index 0000000000000000000000000000000000000000..ab5e159cf476e8e21a05e662be457292e0da9575 --- /dev/null +++ b/apps/android/app/src/main/java/ai/openclaw/android/NodeApp.kt @@ -0,0 +1,26 @@ +package ai.openclaw.android + +import android.app.Application +import android.os.StrictMode + +class NodeApp : Application() { + val runtime: NodeRuntime by lazy { NodeRuntime(this) } + + override fun onCreate() { + super.onCreate() + if (BuildConfig.DEBUG) { + StrictMode.setThreadPolicy( + StrictMode.ThreadPolicy.Builder() + .detectAll() + .penaltyLog() + .build(), + ) + StrictMode.setVmPolicy( + StrictMode.VmPolicy.Builder() + .detectAll() + .penaltyLog() + .build(), + ) + } + } +} diff --git a/apps/android/app/src/main/java/ai/openclaw/android/NodeForegroundService.kt b/apps/android/app/src/main/java/ai/openclaw/android/NodeForegroundService.kt new file mode 100644 index 0000000000000000000000000000000000000000..ee7c8e006747c029f73aa3b6164eb7f78c29f92c --- /dev/null +++ b/apps/android/app/src/main/java/ai/openclaw/android/NodeForegroundService.kt @@ -0,0 +1,180 @@ +package ai.openclaw.android + +import android.app.Notification +import android.app.NotificationChannel +import android.app.NotificationManager +import android.app.Service +import android.app.PendingIntent +import android.Manifest +import android.content.Context +import android.content.Intent +import android.content.pm.PackageManager +import android.content.pm.ServiceInfo +import androidx.core.app.NotificationCompat +import androidx.core.content.ContextCompat +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.Job +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.cancel +import kotlinx.coroutines.flow.combine +import kotlinx.coroutines.launch + +class NodeForegroundService : Service() { + private val scope: CoroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.Main) + private var notificationJob: Job? = null + private var lastRequiresMic = false + private var didStartForeground = false + + override fun onCreate() { + super.onCreate() + ensureChannel() + val initial = buildNotification(title = "OpenClaw Node", text = "Starting…") + startForegroundWithTypes(notification = initial, requiresMic = false) + + val runtime = (application as NodeApp).runtime + notificationJob = + scope.launch { + combine( + runtime.statusText, + runtime.serverName, + runtime.isConnected, + runtime.voiceWakeMode, + runtime.voiceWakeIsListening, + ) { status, server, connected, voiceMode, voiceListening -> + Quint(status, server, connected, voiceMode, voiceListening) + }.collect { (status, server, connected, voiceMode, voiceListening) -> + val title = if (connected) "OpenClaw Node · Connected" else "OpenClaw Node" + val voiceSuffix = + if (voiceMode == VoiceWakeMode.Always) { + if (voiceListening) " · Voice Wake: Listening" else " · Voice Wake: Paused" + } else { + "" + } + val text = (server?.let { "$status · $it" } ?: status) + voiceSuffix + + val requiresMic = + voiceMode == VoiceWakeMode.Always && hasRecordAudioPermission() + startForegroundWithTypes( + notification = buildNotification(title = title, text = text), + requiresMic = requiresMic, + ) + } + } + } + + override fun onStartCommand(intent: Intent?, flags: Int, startId: Int): Int { + when (intent?.action) { + ACTION_STOP -> { + (application as NodeApp).runtime.disconnect() + stopSelf() + return START_NOT_STICKY + } + } + // Keep running; connection is managed by NodeRuntime (auto-reconnect + manual). + return START_STICKY + } + + override fun onDestroy() { + notificationJob?.cancel() + scope.cancel() + super.onDestroy() + } + + override fun onBind(intent: Intent?) = null + + private fun ensureChannel() { + val mgr = getSystemService(NotificationManager::class.java) + val channel = + NotificationChannel( + CHANNEL_ID, + "Connection", + NotificationManager.IMPORTANCE_LOW, + ).apply { + description = "OpenClaw node connection status" + setShowBadge(false) + } + mgr.createNotificationChannel(channel) + } + + private fun buildNotification(title: String, text: String): Notification { + val launchIntent = Intent(this, MainActivity::class.java).apply { + flags = Intent.FLAG_ACTIVITY_SINGLE_TOP or Intent.FLAG_ACTIVITY_CLEAR_TOP + } + val launchPending = + PendingIntent.getActivity( + this, + 1, + launchIntent, + PendingIntent.FLAG_UPDATE_CURRENT or PendingIntent.FLAG_IMMUTABLE, + ) + + val stopIntent = Intent(this, NodeForegroundService::class.java).setAction(ACTION_STOP) + val stopPending = + PendingIntent.getService( + this, + 2, + stopIntent, + PendingIntent.FLAG_UPDATE_CURRENT or PendingIntent.FLAG_IMMUTABLE, + ) + + return NotificationCompat.Builder(this, CHANNEL_ID) + .setSmallIcon(R.mipmap.ic_launcher) + .setContentTitle(title) + .setContentText(text) + .setContentIntent(launchPending) + .setOngoing(true) + .setOnlyAlertOnce(true) + .setForegroundServiceBehavior(NotificationCompat.FOREGROUND_SERVICE_IMMEDIATE) + .addAction(0, "Disconnect", stopPending) + .build() + } + + private fun updateNotification(notification: Notification) { + val mgr = getSystemService(Context.NOTIFICATION_SERVICE) as NotificationManager + mgr.notify(NOTIFICATION_ID, notification) + } + + private fun startForegroundWithTypes(notification: Notification, requiresMic: Boolean) { + if (didStartForeground && requiresMic == lastRequiresMic) { + updateNotification(notification) + return + } + + lastRequiresMic = requiresMic + val types = + if (requiresMic) { + ServiceInfo.FOREGROUND_SERVICE_TYPE_DATA_SYNC or ServiceInfo.FOREGROUND_SERVICE_TYPE_MICROPHONE + } else { + ServiceInfo.FOREGROUND_SERVICE_TYPE_DATA_SYNC + } + startForeground(NOTIFICATION_ID, notification, types) + didStartForeground = true + } + + private fun hasRecordAudioPermission(): Boolean { + return ( + ContextCompat.checkSelfPermission(this, Manifest.permission.RECORD_AUDIO) == + PackageManager.PERMISSION_GRANTED + ) + } + + companion object { + private const val CHANNEL_ID = "connection" + private const val NOTIFICATION_ID = 1 + + private const val ACTION_STOP = "ai.openclaw.android.action.STOP" + + fun start(context: Context) { + val intent = Intent(context, NodeForegroundService::class.java) + context.startForegroundService(intent) + } + + fun stop(context: Context) { + val intent = Intent(context, NodeForegroundService::class.java).setAction(ACTION_STOP) + context.startService(intent) + } + } +} + +private data class Quint(val first: A, val second: B, val third: C, val fourth: D, val fifth: E) diff --git a/apps/android/app/src/main/java/ai/openclaw/android/NodeRuntime.kt b/apps/android/app/src/main/java/ai/openclaw/android/NodeRuntime.kt new file mode 100644 index 0000000000000000000000000000000000000000..e6ceae598d0a14430e26562c19d630c9754b5517 --- /dev/null +++ b/apps/android/app/src/main/java/ai/openclaw/android/NodeRuntime.kt @@ -0,0 +1,1271 @@ +package ai.openclaw.android + +import android.Manifest +import android.content.Context +import android.content.pm.PackageManager +import android.location.LocationManager +import android.os.Build +import android.os.SystemClock +import androidx.core.content.ContextCompat +import ai.openclaw.android.chat.ChatController +import ai.openclaw.android.chat.ChatMessage +import ai.openclaw.android.chat.ChatPendingToolCall +import ai.openclaw.android.chat.ChatSessionEntry +import ai.openclaw.android.chat.OutgoingAttachment +import ai.openclaw.android.gateway.DeviceAuthStore +import ai.openclaw.android.gateway.DeviceIdentityStore +import ai.openclaw.android.gateway.GatewayClientInfo +import ai.openclaw.android.gateway.GatewayConnectOptions +import ai.openclaw.android.gateway.GatewayDiscovery +import ai.openclaw.android.gateway.GatewayEndpoint +import ai.openclaw.android.gateway.GatewaySession +import ai.openclaw.android.gateway.GatewayTlsParams +import ai.openclaw.android.node.CameraCaptureManager +import ai.openclaw.android.node.LocationCaptureManager +import ai.openclaw.android.BuildConfig +import ai.openclaw.android.node.CanvasController +import ai.openclaw.android.node.ScreenRecordManager +import ai.openclaw.android.node.SmsManager +import ai.openclaw.android.protocol.OpenClawCapability +import ai.openclaw.android.protocol.OpenClawCameraCommand +import ai.openclaw.android.protocol.OpenClawCanvasA2UIAction +import ai.openclaw.android.protocol.OpenClawCanvasA2UICommand +import ai.openclaw.android.protocol.OpenClawCanvasCommand +import ai.openclaw.android.protocol.OpenClawScreenCommand +import ai.openclaw.android.protocol.OpenClawLocationCommand +import ai.openclaw.android.protocol.OpenClawSmsCommand +import ai.openclaw.android.voice.TalkModeManager +import ai.openclaw.android.voice.VoiceWakeManager +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.Job +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.TimeoutCancellationException +import kotlinx.coroutines.delay +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.asStateFlow +import kotlinx.coroutines.flow.combine +import kotlinx.coroutines.flow.collect +import kotlinx.coroutines.flow.distinctUntilChanged +import kotlinx.coroutines.launch +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.JsonArray +import kotlinx.serialization.json.JsonElement +import kotlinx.serialization.json.JsonNull +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.JsonPrimitive +import kotlinx.serialization.json.buildJsonObject +import java.util.concurrent.atomic.AtomicLong + +class NodeRuntime(context: Context) { + private val appContext = context.applicationContext + private val scope = CoroutineScope(SupervisorJob() + Dispatchers.IO) + + val prefs = SecurePrefs(appContext) + private val deviceAuthStore = DeviceAuthStore(prefs) + val canvas = CanvasController() + val camera = CameraCaptureManager(appContext) + val location = LocationCaptureManager(appContext) + val screenRecorder = ScreenRecordManager(appContext) + val sms = SmsManager(appContext) + private val json = Json { ignoreUnknownKeys = true } + + private val externalAudioCaptureActive = MutableStateFlow(false) + + private val voiceWake: VoiceWakeManager by lazy { + VoiceWakeManager( + context = appContext, + scope = scope, + onCommand = { command -> + nodeSession.sendNodeEvent( + event = "agent.request", + payloadJson = + buildJsonObject { + put("message", JsonPrimitive(command)) + put("sessionKey", JsonPrimitive(resolveMainSessionKey())) + put("thinking", JsonPrimitive(chatThinkingLevel.value)) + put("deliver", JsonPrimitive(false)) + }.toString(), + ) + }, + ) + } + + val voiceWakeIsListening: StateFlow + get() = voiceWake.isListening + + val voiceWakeStatusText: StateFlow + get() = voiceWake.statusText + + val talkStatusText: StateFlow + get() = talkMode.statusText + + val talkIsListening: StateFlow + get() = talkMode.isListening + + val talkIsSpeaking: StateFlow + get() = talkMode.isSpeaking + + private val discovery = GatewayDiscovery(appContext, scope = scope) + val gateways: StateFlow> = discovery.gateways + val discoveryStatusText: StateFlow = discovery.statusText + + private val identityStore = DeviceIdentityStore(appContext) + + private val _isConnected = MutableStateFlow(false) + val isConnected: StateFlow = _isConnected.asStateFlow() + + private val _statusText = MutableStateFlow("Offline") + val statusText: StateFlow = _statusText.asStateFlow() + + private val _mainSessionKey = MutableStateFlow("main") + val mainSessionKey: StateFlow = _mainSessionKey.asStateFlow() + + private val cameraHudSeq = AtomicLong(0) + private val _cameraHud = MutableStateFlow(null) + val cameraHud: StateFlow = _cameraHud.asStateFlow() + + private val _cameraFlashToken = MutableStateFlow(0L) + val cameraFlashToken: StateFlow = _cameraFlashToken.asStateFlow() + + private val _screenRecordActive = MutableStateFlow(false) + val screenRecordActive: StateFlow = _screenRecordActive.asStateFlow() + + private val _serverName = MutableStateFlow(null) + val serverName: StateFlow = _serverName.asStateFlow() + + private val _remoteAddress = MutableStateFlow(null) + val remoteAddress: StateFlow = _remoteAddress.asStateFlow() + + private val _seamColorArgb = MutableStateFlow(DEFAULT_SEAM_COLOR_ARGB) + val seamColorArgb: StateFlow = _seamColorArgb.asStateFlow() + + private val _isForeground = MutableStateFlow(true) + val isForeground: StateFlow = _isForeground.asStateFlow() + + private var lastAutoA2uiUrl: String? = null + private var operatorConnected = false + private var nodeConnected = false + private var operatorStatusText: String = "Offline" + private var nodeStatusText: String = "Offline" + private var connectedEndpoint: GatewayEndpoint? = null + + private val operatorSession = + GatewaySession( + scope = scope, + identityStore = identityStore, + deviceAuthStore = deviceAuthStore, + onConnected = { name, remote, mainSessionKey -> + operatorConnected = true + operatorStatusText = "Connected" + _serverName.value = name + _remoteAddress.value = remote + _seamColorArgb.value = DEFAULT_SEAM_COLOR_ARGB + applyMainSessionKey(mainSessionKey) + updateStatus() + scope.launch { refreshBrandingFromGateway() } + scope.launch { refreshWakeWordsFromGateway() } + }, + onDisconnected = { message -> + operatorConnected = false + operatorStatusText = message + _serverName.value = null + _remoteAddress.value = null + _seamColorArgb.value = DEFAULT_SEAM_COLOR_ARGB + if (!isCanonicalMainSessionKey(_mainSessionKey.value)) { + _mainSessionKey.value = "main" + } + val mainKey = resolveMainSessionKey() + talkMode.setMainSessionKey(mainKey) + chat.applyMainSessionKey(mainKey) + chat.onDisconnected(message) + updateStatus() + }, + onEvent = { event, payloadJson -> + handleGatewayEvent(event, payloadJson) + }, + ) + + private val nodeSession = + GatewaySession( + scope = scope, + identityStore = identityStore, + deviceAuthStore = deviceAuthStore, + onConnected = { _, _, _ -> + nodeConnected = true + nodeStatusText = "Connected" + updateStatus() + maybeNavigateToA2uiOnConnect() + }, + onDisconnected = { message -> + nodeConnected = false + nodeStatusText = message + updateStatus() + showLocalCanvasOnDisconnect() + }, + onEvent = { _, _ -> }, + onInvoke = { req -> + handleInvoke(req.command, req.paramsJson) + }, + onTlsFingerprint = { stableId, fingerprint -> + prefs.saveGatewayTlsFingerprint(stableId, fingerprint) + }, + ) + + private val chat: ChatController = + ChatController( + scope = scope, + session = operatorSession, + json = json, + supportsChatSubscribe = false, + ) + private val talkMode: TalkModeManager by lazy { + TalkModeManager( + context = appContext, + scope = scope, + session = operatorSession, + supportsChatSubscribe = false, + isConnected = { operatorConnected }, + ) + } + + private fun applyMainSessionKey(candidate: String?) { + val trimmed = candidate?.trim().orEmpty() + if (trimmed.isEmpty()) return + if (isCanonicalMainSessionKey(_mainSessionKey.value)) return + if (_mainSessionKey.value == trimmed) return + _mainSessionKey.value = trimmed + talkMode.setMainSessionKey(trimmed) + chat.applyMainSessionKey(trimmed) + } + + private fun updateStatus() { + _isConnected.value = operatorConnected + _statusText.value = + when { + operatorConnected && nodeConnected -> "Connected" + operatorConnected && !nodeConnected -> "Connected (node offline)" + !operatorConnected && nodeConnected -> "Connected (operator offline)" + operatorStatusText.isNotBlank() && operatorStatusText != "Offline" -> operatorStatusText + else -> nodeStatusText + } + } + + private fun resolveMainSessionKey(): String { + val trimmed = _mainSessionKey.value.trim() + return if (trimmed.isEmpty()) "main" else trimmed + } + + private fun maybeNavigateToA2uiOnConnect() { + val a2uiUrl = resolveA2uiHostUrl() ?: return + val current = canvas.currentUrl()?.trim().orEmpty() + if (current.isEmpty() || current == lastAutoA2uiUrl) { + lastAutoA2uiUrl = a2uiUrl + canvas.navigate(a2uiUrl) + } + } + + private fun showLocalCanvasOnDisconnect() { + lastAutoA2uiUrl = null + canvas.navigate("") + } + + val instanceId: StateFlow = prefs.instanceId + val displayName: StateFlow = prefs.displayName + val cameraEnabled: StateFlow = prefs.cameraEnabled + val locationMode: StateFlow = prefs.locationMode + val locationPreciseEnabled: StateFlow = prefs.locationPreciseEnabled + val preventSleep: StateFlow = prefs.preventSleep + val wakeWords: StateFlow> = prefs.wakeWords + val voiceWakeMode: StateFlow = prefs.voiceWakeMode + val talkEnabled: StateFlow = prefs.talkEnabled + val manualEnabled: StateFlow = prefs.manualEnabled + val manualHost: StateFlow = prefs.manualHost + val manualPort: StateFlow = prefs.manualPort + val manualTls: StateFlow = prefs.manualTls + val lastDiscoveredStableId: StateFlow = prefs.lastDiscoveredStableId + val canvasDebugStatusEnabled: StateFlow = prefs.canvasDebugStatusEnabled + + private var didAutoConnect = false + private var suppressWakeWordsSync = false + private var wakeWordsSyncJob: Job? = null + + val chatSessionKey: StateFlow = chat.sessionKey + val chatSessionId: StateFlow = chat.sessionId + val chatMessages: StateFlow> = chat.messages + val chatError: StateFlow = chat.errorText + val chatHealthOk: StateFlow = chat.healthOk + val chatThinkingLevel: StateFlow = chat.thinkingLevel + val chatStreamingAssistantText: StateFlow = chat.streamingAssistantText + val chatPendingToolCalls: StateFlow> = chat.pendingToolCalls + val chatSessions: StateFlow> = chat.sessions + val pendingRunCount: StateFlow = chat.pendingRunCount + + init { + scope.launch { + combine( + voiceWakeMode, + isForeground, + externalAudioCaptureActive, + wakeWords, + ) { mode, foreground, externalAudio, words -> + Quad(mode, foreground, externalAudio, words) + }.distinctUntilChanged() + .collect { (mode, foreground, externalAudio, words) -> + voiceWake.setTriggerWords(words) + + val shouldListen = + when (mode) { + VoiceWakeMode.Off -> false + VoiceWakeMode.Foreground -> foreground + VoiceWakeMode.Always -> true + } && !externalAudio + + if (!shouldListen) { + voiceWake.stop(statusText = if (mode == VoiceWakeMode.Off) "Off" else "Paused") + return@collect + } + + if (!hasRecordAudioPermission()) { + voiceWake.stop(statusText = "Microphone permission required") + return@collect + } + + voiceWake.start() + } + } + + scope.launch { + talkEnabled.collect { enabled -> + talkMode.setEnabled(enabled) + externalAudioCaptureActive.value = enabled + } + } + + scope.launch(Dispatchers.Default) { + gateways.collect { list -> + if (list.isNotEmpty()) { + // Persist the last discovered gateway (best-effort UX parity with iOS). + prefs.setLastDiscoveredStableId(list.last().stableId) + } + + if (didAutoConnect) return@collect + if (_isConnected.value) return@collect + + if (manualEnabled.value) { + val host = manualHost.value.trim() + val port = manualPort.value + if (host.isNotEmpty() && port in 1..65535) { + didAutoConnect = true + connect(GatewayEndpoint.manual(host = host, port = port)) + } + return@collect + } + + val targetStableId = lastDiscoveredStableId.value.trim() + if (targetStableId.isEmpty()) return@collect + val target = list.firstOrNull { it.stableId == targetStableId } ?: return@collect + didAutoConnect = true + connect(target) + } + } + + scope.launch { + combine( + canvasDebugStatusEnabled, + statusText, + serverName, + remoteAddress, + ) { debugEnabled, status, server, remote -> + Quad(debugEnabled, status, server, remote) + }.distinctUntilChanged() + .collect { (debugEnabled, status, server, remote) -> + canvas.setDebugStatusEnabled(debugEnabled) + if (!debugEnabled) return@collect + canvas.setDebugStatus(status, server ?: remote) + } + } + } + + fun setForeground(value: Boolean) { + _isForeground.value = value + } + + fun setDisplayName(value: String) { + prefs.setDisplayName(value) + } + + fun setCameraEnabled(value: Boolean) { + prefs.setCameraEnabled(value) + } + + fun setLocationMode(mode: LocationMode) { + prefs.setLocationMode(mode) + } + + fun setLocationPreciseEnabled(value: Boolean) { + prefs.setLocationPreciseEnabled(value) + } + + fun setPreventSleep(value: Boolean) { + prefs.setPreventSleep(value) + } + + fun setManualEnabled(value: Boolean) { + prefs.setManualEnabled(value) + } + + fun setManualHost(value: String) { + prefs.setManualHost(value) + } + + fun setManualPort(value: Int) { + prefs.setManualPort(value) + } + + fun setManualTls(value: Boolean) { + prefs.setManualTls(value) + } + + fun setCanvasDebugStatusEnabled(value: Boolean) { + prefs.setCanvasDebugStatusEnabled(value) + } + + fun setWakeWords(words: List) { + prefs.setWakeWords(words) + scheduleWakeWordsSyncIfNeeded() + } + + fun resetWakeWordsDefaults() { + setWakeWords(SecurePrefs.defaultWakeWords) + } + + fun setVoiceWakeMode(mode: VoiceWakeMode) { + prefs.setVoiceWakeMode(mode) + } + + fun setTalkEnabled(value: Boolean) { + prefs.setTalkEnabled(value) + } + + private fun buildInvokeCommands(): List = + buildList { + add(OpenClawCanvasCommand.Present.rawValue) + add(OpenClawCanvasCommand.Hide.rawValue) + add(OpenClawCanvasCommand.Navigate.rawValue) + add(OpenClawCanvasCommand.Eval.rawValue) + add(OpenClawCanvasCommand.Snapshot.rawValue) + add(OpenClawCanvasA2UICommand.Push.rawValue) + add(OpenClawCanvasA2UICommand.PushJSONL.rawValue) + add(OpenClawCanvasA2UICommand.Reset.rawValue) + add(OpenClawScreenCommand.Record.rawValue) + if (cameraEnabled.value) { + add(OpenClawCameraCommand.Snap.rawValue) + add(OpenClawCameraCommand.Clip.rawValue) + } + if (locationMode.value != LocationMode.Off) { + add(OpenClawLocationCommand.Get.rawValue) + } + if (sms.canSendSms()) { + add(OpenClawSmsCommand.Send.rawValue) + } + } + + private fun buildCapabilities(): List = + buildList { + add(OpenClawCapability.Canvas.rawValue) + add(OpenClawCapability.Screen.rawValue) + if (cameraEnabled.value) add(OpenClawCapability.Camera.rawValue) + if (sms.canSendSms()) add(OpenClawCapability.Sms.rawValue) + if (voiceWakeMode.value != VoiceWakeMode.Off && hasRecordAudioPermission()) { + add(OpenClawCapability.VoiceWake.rawValue) + } + if (locationMode.value != LocationMode.Off) { + add(OpenClawCapability.Location.rawValue) + } + } + + private fun resolvedVersionName(): String { + val versionName = BuildConfig.VERSION_NAME.trim().ifEmpty { "dev" } + return if (BuildConfig.DEBUG && !versionName.contains("dev", ignoreCase = true)) { + "$versionName-dev" + } else { + versionName + } + } + + private fun resolveModelIdentifier(): String? { + return listOfNotNull(Build.MANUFACTURER, Build.MODEL) + .joinToString(" ") + .trim() + .ifEmpty { null } + } + + private fun buildUserAgent(): String { + val version = resolvedVersionName() + val release = Build.VERSION.RELEASE?.trim().orEmpty() + val releaseLabel = if (release.isEmpty()) "unknown" else release + return "OpenClawAndroid/$version (Android $releaseLabel; SDK ${Build.VERSION.SDK_INT})" + } + + private fun buildClientInfo(clientId: String, clientMode: String): GatewayClientInfo { + return GatewayClientInfo( + id = clientId, + displayName = displayName.value, + version = resolvedVersionName(), + platform = "android", + mode = clientMode, + instanceId = instanceId.value, + deviceFamily = "Android", + modelIdentifier = resolveModelIdentifier(), + ) + } + + private fun buildNodeConnectOptions(): GatewayConnectOptions { + return GatewayConnectOptions( + role = "node", + scopes = emptyList(), + caps = buildCapabilities(), + commands = buildInvokeCommands(), + permissions = emptyMap(), + client = buildClientInfo(clientId = "openclaw-android", clientMode = "node"), + userAgent = buildUserAgent(), + ) + } + + private fun buildOperatorConnectOptions(): GatewayConnectOptions { + return GatewayConnectOptions( + role = "operator", + scopes = emptyList(), + caps = emptyList(), + commands = emptyList(), + permissions = emptyMap(), + client = buildClientInfo(clientId = "openclaw-control-ui", clientMode = "ui"), + userAgent = buildUserAgent(), + ) + } + + fun refreshGatewayConnection() { + val endpoint = connectedEndpoint ?: return + val token = prefs.loadGatewayToken() + val password = prefs.loadGatewayPassword() + val tls = resolveTlsParams(endpoint) + operatorSession.connect(endpoint, token, password, buildOperatorConnectOptions(), tls) + nodeSession.connect(endpoint, token, password, buildNodeConnectOptions(), tls) + operatorSession.reconnect() + nodeSession.reconnect() + } + + fun connect(endpoint: GatewayEndpoint) { + connectedEndpoint = endpoint + operatorStatusText = "Connecting…" + nodeStatusText = "Connecting…" + updateStatus() + val token = prefs.loadGatewayToken() + val password = prefs.loadGatewayPassword() + val tls = resolveTlsParams(endpoint) + operatorSession.connect(endpoint, token, password, buildOperatorConnectOptions(), tls) + nodeSession.connect(endpoint, token, password, buildNodeConnectOptions(), tls) + } + + private fun hasRecordAudioPermission(): Boolean { + return ( + ContextCompat.checkSelfPermission(appContext, Manifest.permission.RECORD_AUDIO) == + PackageManager.PERMISSION_GRANTED + ) + } + + private fun hasFineLocationPermission(): Boolean { + return ( + ContextCompat.checkSelfPermission(appContext, Manifest.permission.ACCESS_FINE_LOCATION) == + PackageManager.PERMISSION_GRANTED + ) + } + + private fun hasCoarseLocationPermission(): Boolean { + return ( + ContextCompat.checkSelfPermission(appContext, Manifest.permission.ACCESS_COARSE_LOCATION) == + PackageManager.PERMISSION_GRANTED + ) + } + + private fun hasBackgroundLocationPermission(): Boolean { + return ( + ContextCompat.checkSelfPermission(appContext, Manifest.permission.ACCESS_BACKGROUND_LOCATION) == + PackageManager.PERMISSION_GRANTED + ) + } + + fun connectManual() { + val host = manualHost.value.trim() + val port = manualPort.value + if (host.isEmpty() || port <= 0 || port > 65535) { + _statusText.value = "Failed: invalid manual host/port" + return + } + connect(GatewayEndpoint.manual(host = host, port = port)) + } + + fun disconnect() { + connectedEndpoint = null + operatorSession.disconnect() + nodeSession.disconnect() + } + + private fun resolveTlsParams(endpoint: GatewayEndpoint): GatewayTlsParams? { + val stored = prefs.loadGatewayTlsFingerprint(endpoint.stableId) + val hinted = endpoint.tlsEnabled || !endpoint.tlsFingerprintSha256.isNullOrBlank() + val manual = endpoint.stableId.startsWith("manual|") + + if (manual) { + if (!manualTls.value) return null + return GatewayTlsParams( + required = true, + expectedFingerprint = endpoint.tlsFingerprintSha256 ?: stored, + allowTOFU = stored == null, + stableId = endpoint.stableId, + ) + } + + if (hinted) { + return GatewayTlsParams( + required = true, + expectedFingerprint = endpoint.tlsFingerprintSha256 ?: stored, + allowTOFU = stored == null, + stableId = endpoint.stableId, + ) + } + + if (!stored.isNullOrBlank()) { + return GatewayTlsParams( + required = true, + expectedFingerprint = stored, + allowTOFU = false, + stableId = endpoint.stableId, + ) + } + + return null + } + + fun handleCanvasA2UIActionFromWebView(payloadJson: String) { + scope.launch { + val trimmed = payloadJson.trim() + if (trimmed.isEmpty()) return@launch + + val root = + try { + json.parseToJsonElement(trimmed).asObjectOrNull() ?: return@launch + } catch (_: Throwable) { + return@launch + } + + val userActionObj = (root["userAction"] as? JsonObject) ?: root + val actionId = (userActionObj["id"] as? JsonPrimitive)?.content?.trim().orEmpty().ifEmpty { + java.util.UUID.randomUUID().toString() + } + val name = OpenClawCanvasA2UIAction.extractActionName(userActionObj) ?: return@launch + + val surfaceId = + (userActionObj["surfaceId"] as? JsonPrimitive)?.content?.trim().orEmpty().ifEmpty { "main" } + val sourceComponentId = + (userActionObj["sourceComponentId"] as? JsonPrimitive)?.content?.trim().orEmpty().ifEmpty { "-" } + val contextJson = (userActionObj["context"] as? JsonObject)?.toString() + + val sessionKey = resolveMainSessionKey() + val message = + OpenClawCanvasA2UIAction.formatAgentMessage( + actionName = name, + sessionKey = sessionKey, + surfaceId = surfaceId, + sourceComponentId = sourceComponentId, + host = displayName.value, + instanceId = instanceId.value.lowercase(), + contextJson = contextJson, + ) + + val connected = nodeConnected + var error: String? = null + if (connected) { + try { + nodeSession.sendNodeEvent( + event = "agent.request", + payloadJson = + buildJsonObject { + put("message", JsonPrimitive(message)) + put("sessionKey", JsonPrimitive(sessionKey)) + put("thinking", JsonPrimitive("low")) + put("deliver", JsonPrimitive(false)) + put("key", JsonPrimitive(actionId)) + }.toString(), + ) + } catch (e: Throwable) { + error = e.message ?: "send failed" + } + } else { + error = "gateway not connected" + } + + try { + canvas.eval( + OpenClawCanvasA2UIAction.jsDispatchA2UIActionStatus( + actionId = actionId, + ok = connected && error == null, + error = error, + ), + ) + } catch (_: Throwable) { + // ignore + } + } + } + + fun loadChat(sessionKey: String) { + val key = sessionKey.trim().ifEmpty { resolveMainSessionKey() } + chat.load(key) + } + + fun refreshChat() { + chat.refresh() + } + + fun refreshChatSessions(limit: Int? = null) { + chat.refreshSessions(limit = limit) + } + + fun setChatThinkingLevel(level: String) { + chat.setThinkingLevel(level) + } + + fun switchChatSession(sessionKey: String) { + chat.switchSession(sessionKey) + } + + fun abortChat() { + chat.abort() + } + + fun sendChat(message: String, thinking: String, attachments: List) { + chat.sendMessage(message = message, thinkingLevel = thinking, attachments = attachments) + } + + private fun handleGatewayEvent(event: String, payloadJson: String?) { + if (event == "voicewake.changed") { + if (payloadJson.isNullOrBlank()) return + try { + val payload = json.parseToJsonElement(payloadJson).asObjectOrNull() ?: return + val array = payload["triggers"] as? JsonArray ?: return + val triggers = array.mapNotNull { it.asStringOrNull() } + applyWakeWordsFromGateway(triggers) + } catch (_: Throwable) { + // ignore + } + return + } + + talkMode.handleGatewayEvent(event, payloadJson) + chat.handleGatewayEvent(event, payloadJson) + } + + private fun applyWakeWordsFromGateway(words: List) { + suppressWakeWordsSync = true + prefs.setWakeWords(words) + suppressWakeWordsSync = false + } + + private fun scheduleWakeWordsSyncIfNeeded() { + if (suppressWakeWordsSync) return + if (!_isConnected.value) return + + val snapshot = prefs.wakeWords.value + wakeWordsSyncJob?.cancel() + wakeWordsSyncJob = + scope.launch { + delay(650) + val jsonList = snapshot.joinToString(separator = ",") { it.toJsonString() } + val params = """{"triggers":[$jsonList]}""" + try { + operatorSession.request("voicewake.set", params) + } catch (_: Throwable) { + // ignore + } + } + } + + private suspend fun refreshWakeWordsFromGateway() { + if (!_isConnected.value) return + try { + val res = operatorSession.request("voicewake.get", "{}") + val payload = json.parseToJsonElement(res).asObjectOrNull() ?: return + val array = payload["triggers"] as? JsonArray ?: return + val triggers = array.mapNotNull { it.asStringOrNull() } + applyWakeWordsFromGateway(triggers) + } catch (_: Throwable) { + // ignore + } + } + + private suspend fun refreshBrandingFromGateway() { + if (!_isConnected.value) return + try { + val res = operatorSession.request("config.get", "{}") + val root = json.parseToJsonElement(res).asObjectOrNull() + val config = root?.get("config").asObjectOrNull() + val ui = config?.get("ui").asObjectOrNull() + val raw = ui?.get("seamColor").asStringOrNull()?.trim() + val sessionCfg = config?.get("session").asObjectOrNull() + val mainKey = normalizeMainKey(sessionCfg?.get("mainKey").asStringOrNull()) + applyMainSessionKey(mainKey) + + val parsed = parseHexColorArgb(raw) + _seamColorArgb.value = parsed ?: DEFAULT_SEAM_COLOR_ARGB + } catch (_: Throwable) { + // ignore + } + } + + private suspend fun handleInvoke(command: String, paramsJson: String?): GatewaySession.InvokeResult { + if ( + command.startsWith(OpenClawCanvasCommand.NamespacePrefix) || + command.startsWith(OpenClawCanvasA2UICommand.NamespacePrefix) || + command.startsWith(OpenClawCameraCommand.NamespacePrefix) || + command.startsWith(OpenClawScreenCommand.NamespacePrefix) + ) { + if (!isForeground.value) { + return GatewaySession.InvokeResult.error( + code = "NODE_BACKGROUND_UNAVAILABLE", + message = "NODE_BACKGROUND_UNAVAILABLE: canvas/camera/screen commands require foreground", + ) + } + } + if (command.startsWith(OpenClawCameraCommand.NamespacePrefix) && !cameraEnabled.value) { + return GatewaySession.InvokeResult.error( + code = "CAMERA_DISABLED", + message = "CAMERA_DISABLED: enable Camera in Settings", + ) + } + if (command.startsWith(OpenClawLocationCommand.NamespacePrefix) && + locationMode.value == LocationMode.Off + ) { + return GatewaySession.InvokeResult.error( + code = "LOCATION_DISABLED", + message = "LOCATION_DISABLED: enable Location in Settings", + ) + } + + return when (command) { + OpenClawCanvasCommand.Present.rawValue -> { + val url = CanvasController.parseNavigateUrl(paramsJson) + canvas.navigate(url) + GatewaySession.InvokeResult.ok(null) + } + OpenClawCanvasCommand.Hide.rawValue -> GatewaySession.InvokeResult.ok(null) + OpenClawCanvasCommand.Navigate.rawValue -> { + val url = CanvasController.parseNavigateUrl(paramsJson) + canvas.navigate(url) + GatewaySession.InvokeResult.ok(null) + } + OpenClawCanvasCommand.Eval.rawValue -> { + val js = + CanvasController.parseEvalJs(paramsJson) + ?: return GatewaySession.InvokeResult.error( + code = "INVALID_REQUEST", + message = "INVALID_REQUEST: javaScript required", + ) + val result = + try { + canvas.eval(js) + } catch (err: Throwable) { + return GatewaySession.InvokeResult.error( + code = "NODE_BACKGROUND_UNAVAILABLE", + message = "NODE_BACKGROUND_UNAVAILABLE: canvas unavailable", + ) + } + GatewaySession.InvokeResult.ok("""{"result":${result.toJsonString()}}""") + } + OpenClawCanvasCommand.Snapshot.rawValue -> { + val snapshotParams = CanvasController.parseSnapshotParams(paramsJson) + val base64 = + try { + canvas.snapshotBase64( + format = snapshotParams.format, + quality = snapshotParams.quality, + maxWidth = snapshotParams.maxWidth, + ) + } catch (err: Throwable) { + return GatewaySession.InvokeResult.error( + code = "NODE_BACKGROUND_UNAVAILABLE", + message = "NODE_BACKGROUND_UNAVAILABLE: canvas unavailable", + ) + } + GatewaySession.InvokeResult.ok("""{"format":"${snapshotParams.format.rawValue}","base64":"$base64"}""") + } + OpenClawCanvasA2UICommand.Reset.rawValue -> { + val a2uiUrl = resolveA2uiHostUrl() + ?: return GatewaySession.InvokeResult.error( + code = "A2UI_HOST_NOT_CONFIGURED", + message = "A2UI_HOST_NOT_CONFIGURED: gateway did not advertise canvas host", + ) + val ready = ensureA2uiReady(a2uiUrl) + if (!ready) { + return GatewaySession.InvokeResult.error( + code = "A2UI_HOST_UNAVAILABLE", + message = "A2UI host not reachable", + ) + } + val res = canvas.eval(a2uiResetJS) + GatewaySession.InvokeResult.ok(res) + } + OpenClawCanvasA2UICommand.Push.rawValue, OpenClawCanvasA2UICommand.PushJSONL.rawValue -> { + val messages = + try { + decodeA2uiMessages(command, paramsJson) + } catch (err: Throwable) { + return GatewaySession.InvokeResult.error(code = "INVALID_REQUEST", message = err.message ?: "invalid A2UI payload") + } + val a2uiUrl = resolveA2uiHostUrl() + ?: return GatewaySession.InvokeResult.error( + code = "A2UI_HOST_NOT_CONFIGURED", + message = "A2UI_HOST_NOT_CONFIGURED: gateway did not advertise canvas host", + ) + val ready = ensureA2uiReady(a2uiUrl) + if (!ready) { + return GatewaySession.InvokeResult.error( + code = "A2UI_HOST_UNAVAILABLE", + message = "A2UI host not reachable", + ) + } + val js = a2uiApplyMessagesJS(messages) + val res = canvas.eval(js) + GatewaySession.InvokeResult.ok(res) + } + OpenClawCameraCommand.Snap.rawValue -> { + showCameraHud(message = "Taking photo…", kind = CameraHudKind.Photo) + triggerCameraFlash() + val res = + try { + camera.snap(paramsJson) + } catch (err: Throwable) { + val (code, message) = invokeErrorFromThrowable(err) + showCameraHud(message = message, kind = CameraHudKind.Error, autoHideMs = 2200) + return GatewaySession.InvokeResult.error(code = code, message = message) + } + showCameraHud(message = "Photo captured", kind = CameraHudKind.Success, autoHideMs = 1600) + GatewaySession.InvokeResult.ok(res.payloadJson) + } + OpenClawCameraCommand.Clip.rawValue -> { + val includeAudio = paramsJson?.contains("\"includeAudio\":true") != false + if (includeAudio) externalAudioCaptureActive.value = true + try { + showCameraHud(message = "Recording…", kind = CameraHudKind.Recording) + val res = + try { + camera.clip(paramsJson) + } catch (err: Throwable) { + val (code, message) = invokeErrorFromThrowable(err) + showCameraHud(message = message, kind = CameraHudKind.Error, autoHideMs = 2400) + return GatewaySession.InvokeResult.error(code = code, message = message) + } + showCameraHud(message = "Clip captured", kind = CameraHudKind.Success, autoHideMs = 1800) + GatewaySession.InvokeResult.ok(res.payloadJson) + } finally { + if (includeAudio) externalAudioCaptureActive.value = false + } + } + OpenClawLocationCommand.Get.rawValue -> { + val mode = locationMode.value + if (!isForeground.value && mode != LocationMode.Always) { + return GatewaySession.InvokeResult.error( + code = "LOCATION_BACKGROUND_UNAVAILABLE", + message = "LOCATION_BACKGROUND_UNAVAILABLE: background location requires Always", + ) + } + if (!hasFineLocationPermission() && !hasCoarseLocationPermission()) { + return GatewaySession.InvokeResult.error( + code = "LOCATION_PERMISSION_REQUIRED", + message = "LOCATION_PERMISSION_REQUIRED: grant Location permission", + ) + } + if (!isForeground.value && mode == LocationMode.Always && !hasBackgroundLocationPermission()) { + return GatewaySession.InvokeResult.error( + code = "LOCATION_PERMISSION_REQUIRED", + message = "LOCATION_PERMISSION_REQUIRED: enable Always in system Settings", + ) + } + val (maxAgeMs, timeoutMs, desiredAccuracy) = parseLocationParams(paramsJson) + val preciseEnabled = locationPreciseEnabled.value + val accuracy = + when (desiredAccuracy) { + "precise" -> if (preciseEnabled && hasFineLocationPermission()) "precise" else "balanced" + "coarse" -> "coarse" + else -> if (preciseEnabled && hasFineLocationPermission()) "precise" else "balanced" + } + val providers = + when (accuracy) { + "precise" -> listOf(LocationManager.GPS_PROVIDER, LocationManager.NETWORK_PROVIDER) + "coarse" -> listOf(LocationManager.NETWORK_PROVIDER, LocationManager.GPS_PROVIDER) + else -> listOf(LocationManager.NETWORK_PROVIDER, LocationManager.GPS_PROVIDER) + } + try { + val payload = + location.getLocation( + desiredProviders = providers, + maxAgeMs = maxAgeMs, + timeoutMs = timeoutMs, + isPrecise = accuracy == "precise", + ) + GatewaySession.InvokeResult.ok(payload.payloadJson) + } catch (err: TimeoutCancellationException) { + GatewaySession.InvokeResult.error( + code = "LOCATION_TIMEOUT", + message = "LOCATION_TIMEOUT: no fix in time", + ) + } catch (err: Throwable) { + val message = err.message ?: "LOCATION_UNAVAILABLE: no fix" + GatewaySession.InvokeResult.error(code = "LOCATION_UNAVAILABLE", message = message) + } + } + OpenClawScreenCommand.Record.rawValue -> { + // Status pill mirrors screen recording state so it stays visible without overlay stacking. + _screenRecordActive.value = true + try { + val res = + try { + screenRecorder.record(paramsJson) + } catch (err: Throwable) { + val (code, message) = invokeErrorFromThrowable(err) + return GatewaySession.InvokeResult.error(code = code, message = message) + } + GatewaySession.InvokeResult.ok(res.payloadJson) + } finally { + _screenRecordActive.value = false + } + } + OpenClawSmsCommand.Send.rawValue -> { + val res = sms.send(paramsJson) + if (res.ok) { + GatewaySession.InvokeResult.ok(res.payloadJson) + } else { + val error = res.error ?: "SMS_SEND_FAILED" + val idx = error.indexOf(':') + val code = if (idx > 0) error.substring(0, idx).trim() else "SMS_SEND_FAILED" + GatewaySession.InvokeResult.error(code = code, message = error) + } + } + else -> + GatewaySession.InvokeResult.error( + code = "INVALID_REQUEST", + message = "INVALID_REQUEST: unknown command", + ) + } + } + + private fun triggerCameraFlash() { + // Token is used as a pulse trigger; value doesn't matter as long as it changes. + _cameraFlashToken.value = SystemClock.elapsedRealtimeNanos() + } + + private fun showCameraHud(message: String, kind: CameraHudKind, autoHideMs: Long? = null) { + val token = cameraHudSeq.incrementAndGet() + _cameraHud.value = CameraHudState(token = token, kind = kind, message = message) + + if (autoHideMs != null && autoHideMs > 0) { + scope.launch { + delay(autoHideMs) + if (_cameraHud.value?.token == token) _cameraHud.value = null + } + } + } + + private fun invokeErrorFromThrowable(err: Throwable): Pair { + val raw = (err.message ?: "").trim() + if (raw.isEmpty()) return "UNAVAILABLE" to "UNAVAILABLE: camera error" + + val idx = raw.indexOf(':') + if (idx <= 0) return "UNAVAILABLE" to raw + val code = raw.substring(0, idx).trim().ifEmpty { "UNAVAILABLE" } + val message = raw.substring(idx + 1).trim().ifEmpty { raw } + // Preserve full string for callers/logging, but keep the returned message human-friendly. + return code to "$code: $message" + } + + private fun parseLocationParams(paramsJson: String?): Triple { + if (paramsJson.isNullOrBlank()) { + return Triple(null, 10_000L, null) + } + val root = + try { + json.parseToJsonElement(paramsJson).asObjectOrNull() + } catch (_: Throwable) { + null + } + val maxAgeMs = (root?.get("maxAgeMs") as? JsonPrimitive)?.content?.toLongOrNull() + val timeoutMs = + (root?.get("timeoutMs") as? JsonPrimitive)?.content?.toLongOrNull()?.coerceIn(1_000L, 60_000L) + ?: 10_000L + val desiredAccuracy = + (root?.get("desiredAccuracy") as? JsonPrimitive)?.content?.trim()?.lowercase() + return Triple(maxAgeMs, timeoutMs, desiredAccuracy) + } + + private fun resolveA2uiHostUrl(): String? { + val nodeRaw = nodeSession.currentCanvasHostUrl()?.trim().orEmpty() + val operatorRaw = operatorSession.currentCanvasHostUrl()?.trim().orEmpty() + val raw = if (nodeRaw.isNotBlank()) nodeRaw else operatorRaw + if (raw.isBlank()) return null + val base = raw.trimEnd('/') + return "${base}/__openclaw__/a2ui/?platform=android" + } + + private suspend fun ensureA2uiReady(a2uiUrl: String): Boolean { + try { + val already = canvas.eval(a2uiReadyCheckJS) + if (already == "true") return true + } catch (_: Throwable) { + // ignore + } + + canvas.navigate(a2uiUrl) + repeat(50) { + try { + val ready = canvas.eval(a2uiReadyCheckJS) + if (ready == "true") return true + } catch (_: Throwable) { + // ignore + } + delay(120) + } + return false + } + + private fun decodeA2uiMessages(command: String, paramsJson: String?): String { + val raw = paramsJson?.trim().orEmpty() + if (raw.isBlank()) throw IllegalArgumentException("INVALID_REQUEST: paramsJSON required") + + val obj = + json.parseToJsonElement(raw) as? JsonObject + ?: throw IllegalArgumentException("INVALID_REQUEST: expected object params") + + val jsonlField = (obj["jsonl"] as? JsonPrimitive)?.content?.trim().orEmpty() + val hasMessagesArray = obj["messages"] is JsonArray + + if (command == OpenClawCanvasA2UICommand.PushJSONL.rawValue || (!hasMessagesArray && jsonlField.isNotBlank())) { + val jsonl = jsonlField + if (jsonl.isBlank()) throw IllegalArgumentException("INVALID_REQUEST: jsonl required") + val messages = + jsonl + .lineSequence() + .map { it.trim() } + .filter { it.isNotBlank() } + .mapIndexed { idx, line -> + val el = json.parseToJsonElement(line) + val msg = + el as? JsonObject + ?: throw IllegalArgumentException("A2UI JSONL line ${idx + 1}: expected a JSON object") + validateA2uiV0_8(msg, idx + 1) + msg + } + .toList() + return JsonArray(messages).toString() + } + + val arr = obj["messages"] as? JsonArray ?: throw IllegalArgumentException("INVALID_REQUEST: messages[] required") + val out = + arr.mapIndexed { idx, el -> + val msg = + el as? JsonObject + ?: throw IllegalArgumentException("A2UI messages[${idx}]: expected a JSON object") + validateA2uiV0_8(msg, idx + 1) + msg + } + return JsonArray(out).toString() + } + + private fun validateA2uiV0_8(msg: JsonObject, lineNumber: Int) { + if (msg.containsKey("createSurface")) { + throw IllegalArgumentException( + "A2UI JSONL line $lineNumber: looks like A2UI v0.9 (`createSurface`). Canvas supports v0.8 messages only.", + ) + } + val allowed = setOf("beginRendering", "surfaceUpdate", "dataModelUpdate", "deleteSurface") + val matched = msg.keys.filter { allowed.contains(it) } + if (matched.size != 1) { + val found = msg.keys.sorted().joinToString(", ") + throw IllegalArgumentException( + "A2UI JSONL line $lineNumber: expected exactly one of ${allowed.sorted().joinToString(", ")}; found: $found", + ) + } + } +} + +private data class Quad(val first: A, val second: B, val third: C, val fourth: D) + +private const val DEFAULT_SEAM_COLOR_ARGB: Long = 0xFF4F7A9A + +private const val a2uiReadyCheckJS: String = + """ + (() => { + try { + const host = globalThis.openclawA2UI; + return !!host && typeof host.applyMessages === 'function'; + } catch (_) { + return false; + } + })() + """ + +private const val a2uiResetJS: String = + """ + (() => { + try { + const host = globalThis.openclawA2UI; + if (!host) return { ok: false, error: "missing openclawA2UI" }; + return host.reset(); + } catch (e) { + return { ok: false, error: String(e?.message ?? e) }; + } + })() + """ + +private fun a2uiApplyMessagesJS(messagesJson: String): String { + return """ + (() => { + try { + const host = globalThis.openclawA2UI; + if (!host) return { ok: false, error: "missing openclawA2UI" }; + const messages = $messagesJson; + return host.applyMessages(messages); + } catch (e) { + return { ok: false, error: String(e?.message ?? e) }; + } + })() + """.trimIndent() +} + +private fun String.toJsonString(): String { + val escaped = + this.replace("\\", "\\\\") + .replace("\"", "\\\"") + .replace("\n", "\\n") + .replace("\r", "\\r") + return "\"$escaped\"" +} + +private fun JsonElement?.asObjectOrNull(): JsonObject? = this as? JsonObject + +private fun JsonElement?.asStringOrNull(): String? = + when (this) { + is JsonNull -> null + is JsonPrimitive -> content + else -> null + } + +private fun parseHexColorArgb(raw: String?): Long? { + val trimmed = raw?.trim().orEmpty() + if (trimmed.isEmpty()) return null + val hex = if (trimmed.startsWith("#")) trimmed.drop(1) else trimmed + if (hex.length != 6) return null + val rgb = hex.toLongOrNull(16) ?: return null + return 0xFF000000L or rgb +} diff --git a/apps/android/app/src/main/java/ai/openclaw/android/PermissionRequester.kt b/apps/android/app/src/main/java/ai/openclaw/android/PermissionRequester.kt new file mode 100644 index 0000000000000000000000000000000000000000..0ee267b5588cb39c87d785a95b26dd2eb2013b64 --- /dev/null +++ b/apps/android/app/src/main/java/ai/openclaw/android/PermissionRequester.kt @@ -0,0 +1,133 @@ +package ai.openclaw.android + +import android.content.pm.PackageManager +import android.content.Intent +import android.Manifest +import android.net.Uri +import android.provider.Settings +import androidx.appcompat.app.AlertDialog +import androidx.activity.ComponentActivity +import androidx.activity.result.ActivityResultLauncher +import androidx.activity.result.contract.ActivityResultContracts +import androidx.core.content.ContextCompat +import androidx.core.app.ActivityCompat +import kotlinx.coroutines.CompletableDeferred +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock +import kotlinx.coroutines.withContext +import kotlinx.coroutines.suspendCancellableCoroutine +import kotlin.coroutines.resume + +class PermissionRequester(private val activity: ComponentActivity) { + private val mutex = Mutex() + private var pending: CompletableDeferred>? = null + + private val launcher: ActivityResultLauncher> = + activity.registerForActivityResult(ActivityResultContracts.RequestMultiplePermissions()) { result -> + val p = pending + pending = null + p?.complete(result) + } + + suspend fun requestIfMissing( + permissions: List, + timeoutMs: Long = 20_000, + ): Map = + mutex.withLock { + val missing = + permissions.filter { perm -> + ContextCompat.checkSelfPermission(activity, perm) != PackageManager.PERMISSION_GRANTED + } + if (missing.isEmpty()) { + return permissions.associateWith { true } + } + + val needsRationale = + missing.any { ActivityCompat.shouldShowRequestPermissionRationale(activity, it) } + if (needsRationale) { + val proceed = showRationaleDialog(missing) + if (!proceed) { + return permissions.associateWith { perm -> + ContextCompat.checkSelfPermission(activity, perm) == PackageManager.PERMISSION_GRANTED + } + } + } + + val deferred = CompletableDeferred>() + pending = deferred + withContext(Dispatchers.Main) { + launcher.launch(missing.toTypedArray()) + } + + val result = + withContext(Dispatchers.Default) { + kotlinx.coroutines.withTimeout(timeoutMs) { deferred.await() } + } + + // Merge: if something was already granted, treat it as granted even if launcher omitted it. + val merged = + permissions.associateWith { perm -> + val nowGranted = + ContextCompat.checkSelfPermission(activity, perm) == PackageManager.PERMISSION_GRANTED + result[perm] == true || nowGranted + } + + val denied = + merged.filterValues { !it }.keys.filter { + !ActivityCompat.shouldShowRequestPermissionRationale(activity, it) + } + if (denied.isNotEmpty()) { + showSettingsDialog(denied) + } + + return merged + } + + private suspend fun showRationaleDialog(permissions: List): Boolean = + withContext(Dispatchers.Main) { + suspendCancellableCoroutine { cont -> + AlertDialog.Builder(activity) + .setTitle("Permission required") + .setMessage(buildRationaleMessage(permissions)) + .setPositiveButton("Continue") { _, _ -> cont.resume(true) } + .setNegativeButton("Not now") { _, _ -> cont.resume(false) } + .setOnCancelListener { cont.resume(false) } + .show() + } + } + + private fun showSettingsDialog(permissions: List) { + AlertDialog.Builder(activity) + .setTitle("Enable permission in Settings") + .setMessage(buildSettingsMessage(permissions)) + .setPositiveButton("Open Settings") { _, _ -> + val intent = + Intent( + Settings.ACTION_APPLICATION_DETAILS_SETTINGS, + Uri.fromParts("package", activity.packageName, null), + ) + activity.startActivity(intent) + } + .setNegativeButton("Cancel", null) + .show() + } + + private fun buildRationaleMessage(permissions: List): String { + val labels = permissions.map { permissionLabel(it) } + return "OpenClaw needs ${labels.joinToString(", ")} permissions to continue." + } + + private fun buildSettingsMessage(permissions: List): String { + val labels = permissions.map { permissionLabel(it) } + return "Please enable ${labels.joinToString(", ")} in Android Settings to continue." + } + + private fun permissionLabel(permission: String): String = + when (permission) { + Manifest.permission.CAMERA -> "Camera" + Manifest.permission.RECORD_AUDIO -> "Microphone" + Manifest.permission.SEND_SMS -> "SMS" + else -> permission + } +} diff --git a/apps/android/app/src/main/java/ai/openclaw/android/ScreenCaptureRequester.kt b/apps/android/app/src/main/java/ai/openclaw/android/ScreenCaptureRequester.kt new file mode 100644 index 0000000000000000000000000000000000000000..c215103b54d422ebaceabb666401ecc14054bc19 --- /dev/null +++ b/apps/android/app/src/main/java/ai/openclaw/android/ScreenCaptureRequester.kt @@ -0,0 +1,65 @@ +package ai.openclaw.android + +import android.app.Activity +import android.content.Context +import android.content.Intent +import android.media.projection.MediaProjectionManager +import androidx.activity.ComponentActivity +import androidx.activity.result.ActivityResultLauncher +import androidx.activity.result.contract.ActivityResultContracts +import androidx.appcompat.app.AlertDialog +import kotlinx.coroutines.CompletableDeferred +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock +import kotlinx.coroutines.withContext +import kotlinx.coroutines.withTimeout +import kotlinx.coroutines.suspendCancellableCoroutine +import kotlin.coroutines.resume + +class ScreenCaptureRequester(private val activity: ComponentActivity) { + data class CaptureResult(val resultCode: Int, val data: Intent) + + private val mutex = Mutex() + private var pending: CompletableDeferred? = null + + private val launcher: ActivityResultLauncher = + activity.registerForActivityResult(ActivityResultContracts.StartActivityForResult()) { result -> + val p = pending + pending = null + val data = result.data + if (result.resultCode == Activity.RESULT_OK && data != null) { + p?.complete(CaptureResult(result.resultCode, data)) + } else { + p?.complete(null) + } + } + + suspend fun requestCapture(timeoutMs: Long = 20_000): CaptureResult? = + mutex.withLock { + val proceed = showRationaleDialog() + if (!proceed) return null + + val mgr = activity.getSystemService(Context.MEDIA_PROJECTION_SERVICE) as MediaProjectionManager + val intent = mgr.createScreenCaptureIntent() + + val deferred = CompletableDeferred() + pending = deferred + withContext(Dispatchers.Main) { launcher.launch(intent) } + + withContext(Dispatchers.Default) { withTimeout(timeoutMs) { deferred.await() } } + } + + private suspend fun showRationaleDialog(): Boolean = + withContext(Dispatchers.Main) { + suspendCancellableCoroutine { cont -> + AlertDialog.Builder(activity) + .setTitle("Screen recording required") + .setMessage("OpenClaw needs to record the screen for this command.") + .setPositiveButton("Continue") { _, _ -> cont.resume(true) } + .setNegativeButton("Not now") { _, _ -> cont.resume(false) } + .setOnCancelListener { cont.resume(false) } + .show() + } + } +} diff --git a/apps/android/app/src/main/java/ai/openclaw/android/SecurePrefs.kt b/apps/android/app/src/main/java/ai/openclaw/android/SecurePrefs.kt new file mode 100644 index 0000000000000000000000000000000000000000..881d724fd142fe9a519d4e91d1271c2aa9be92a8 --- /dev/null +++ b/apps/android/app/src/main/java/ai/openclaw/android/SecurePrefs.kt @@ -0,0 +1,274 @@ +@file:Suppress("DEPRECATION") + +package ai.openclaw.android + +import android.content.Context +import android.content.SharedPreferences +import androidx.core.content.edit +import androidx.security.crypto.EncryptedSharedPreferences +import androidx.security.crypto.MasterKey +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.StateFlow +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.JsonArray +import kotlinx.serialization.json.JsonNull +import kotlinx.serialization.json.JsonPrimitive +import java.util.UUID + +class SecurePrefs(context: Context) { + companion object { + val defaultWakeWords: List = listOf("openclaw", "claude") + private const val displayNameKey = "node.displayName" + private const val voiceWakeModeKey = "voiceWake.mode" + } + + private val appContext = context.applicationContext + private val json = Json { ignoreUnknownKeys = true } + + private val masterKey = + MasterKey.Builder(context) + .setKeyScheme(MasterKey.KeyScheme.AES256_GCM) + .build() + + private val prefs: SharedPreferences by lazy { + createPrefs(appContext, "openclaw.node.secure") + } + + private val _instanceId = MutableStateFlow(loadOrCreateInstanceId()) + val instanceId: StateFlow = _instanceId + + private val _displayName = + MutableStateFlow(loadOrMigrateDisplayName(context = context)) + val displayName: StateFlow = _displayName + + private val _cameraEnabled = MutableStateFlow(prefs.getBoolean("camera.enabled", true)) + val cameraEnabled: StateFlow = _cameraEnabled + + private val _locationMode = + MutableStateFlow(LocationMode.fromRawValue(prefs.getString("location.enabledMode", "off"))) + val locationMode: StateFlow = _locationMode + + private val _locationPreciseEnabled = + MutableStateFlow(prefs.getBoolean("location.preciseEnabled", true)) + val locationPreciseEnabled: StateFlow = _locationPreciseEnabled + + private val _preventSleep = MutableStateFlow(prefs.getBoolean("screen.preventSleep", true)) + val preventSleep: StateFlow = _preventSleep + + private val _manualEnabled = + MutableStateFlow(prefs.getBoolean("gateway.manual.enabled", false)) + val manualEnabled: StateFlow = _manualEnabled + + private val _manualHost = + MutableStateFlow(prefs.getString("gateway.manual.host", "") ?: "") + val manualHost: StateFlow = _manualHost + + private val _manualPort = + MutableStateFlow(prefs.getInt("gateway.manual.port", 18789)) + val manualPort: StateFlow = _manualPort + + private val _manualTls = + MutableStateFlow(prefs.getBoolean("gateway.manual.tls", true)) + val manualTls: StateFlow = _manualTls + + private val _lastDiscoveredStableId = + MutableStateFlow( + prefs.getString("gateway.lastDiscoveredStableID", "") ?: "", + ) + val lastDiscoveredStableId: StateFlow = _lastDiscoveredStableId + + private val _canvasDebugStatusEnabled = + MutableStateFlow(prefs.getBoolean("canvas.debugStatusEnabled", false)) + val canvasDebugStatusEnabled: StateFlow = _canvasDebugStatusEnabled + + private val _wakeWords = MutableStateFlow(loadWakeWords()) + val wakeWords: StateFlow> = _wakeWords + + private val _voiceWakeMode = MutableStateFlow(loadVoiceWakeMode()) + val voiceWakeMode: StateFlow = _voiceWakeMode + + private val _talkEnabled = MutableStateFlow(prefs.getBoolean("talk.enabled", false)) + val talkEnabled: StateFlow = _talkEnabled + + fun setLastDiscoveredStableId(value: String) { + val trimmed = value.trim() + prefs.edit { putString("gateway.lastDiscoveredStableID", trimmed) } + _lastDiscoveredStableId.value = trimmed + } + + fun setDisplayName(value: String) { + val trimmed = value.trim() + prefs.edit { putString(displayNameKey, trimmed) } + _displayName.value = trimmed + } + + fun setCameraEnabled(value: Boolean) { + prefs.edit { putBoolean("camera.enabled", value) } + _cameraEnabled.value = value + } + + fun setLocationMode(mode: LocationMode) { + prefs.edit { putString("location.enabledMode", mode.rawValue) } + _locationMode.value = mode + } + + fun setLocationPreciseEnabled(value: Boolean) { + prefs.edit { putBoolean("location.preciseEnabled", value) } + _locationPreciseEnabled.value = value + } + + fun setPreventSleep(value: Boolean) { + prefs.edit { putBoolean("screen.preventSleep", value) } + _preventSleep.value = value + } + + fun setManualEnabled(value: Boolean) { + prefs.edit { putBoolean("gateway.manual.enabled", value) } + _manualEnabled.value = value + } + + fun setManualHost(value: String) { + val trimmed = value.trim() + prefs.edit { putString("gateway.manual.host", trimmed) } + _manualHost.value = trimmed + } + + fun setManualPort(value: Int) { + prefs.edit { putInt("gateway.manual.port", value) } + _manualPort.value = value + } + + fun setManualTls(value: Boolean) { + prefs.edit { putBoolean("gateway.manual.tls", value) } + _manualTls.value = value + } + + fun setCanvasDebugStatusEnabled(value: Boolean) { + prefs.edit { putBoolean("canvas.debugStatusEnabled", value) } + _canvasDebugStatusEnabled.value = value + } + + fun loadGatewayToken(): String? { + val key = "gateway.token.${_instanceId.value}" + val stored = prefs.getString(key, null)?.trim() + return stored?.takeIf { it.isNotEmpty() } + } + + fun saveGatewayToken(token: String) { + val key = "gateway.token.${_instanceId.value}" + prefs.edit { putString(key, token.trim()) } + } + + fun loadGatewayPassword(): String? { + val key = "gateway.password.${_instanceId.value}" + val stored = prefs.getString(key, null)?.trim() + return stored?.takeIf { it.isNotEmpty() } + } + + fun saveGatewayPassword(password: String) { + val key = "gateway.password.${_instanceId.value}" + prefs.edit { putString(key, password.trim()) } + } + + fun loadGatewayTlsFingerprint(stableId: String): String? { + val key = "gateway.tls.$stableId" + return prefs.getString(key, null)?.trim()?.takeIf { it.isNotEmpty() } + } + + fun saveGatewayTlsFingerprint(stableId: String, fingerprint: String) { + val key = "gateway.tls.$stableId" + prefs.edit { putString(key, fingerprint.trim()) } + } + + fun getString(key: String): String? { + return prefs.getString(key, null) + } + + fun putString(key: String, value: String) { + prefs.edit { putString(key, value) } + } + + fun remove(key: String) { + prefs.edit { remove(key) } + } + + private fun createPrefs(context: Context, name: String): SharedPreferences { + return EncryptedSharedPreferences.create( + context, + name, + masterKey, + EncryptedSharedPreferences.PrefKeyEncryptionScheme.AES256_SIV, + EncryptedSharedPreferences.PrefValueEncryptionScheme.AES256_GCM, + ) + } + + private fun loadOrCreateInstanceId(): String { + val existing = prefs.getString("node.instanceId", null)?.trim() + if (!existing.isNullOrBlank()) return existing + val fresh = UUID.randomUUID().toString() + prefs.edit { putString("node.instanceId", fresh) } + return fresh + } + + private fun loadOrMigrateDisplayName(context: Context): String { + val existing = prefs.getString(displayNameKey, null)?.trim().orEmpty() + if (existing.isNotEmpty() && existing != "Android Node") return existing + + val candidate = DeviceNames.bestDefaultNodeName(context).trim() + val resolved = candidate.ifEmpty { "Android Node" } + + prefs.edit { putString(displayNameKey, resolved) } + return resolved + } + + fun setWakeWords(words: List) { + val sanitized = WakeWords.sanitize(words, defaultWakeWords) + val encoded = + JsonArray(sanitized.map { JsonPrimitive(it) }).toString() + prefs.edit { putString("voiceWake.triggerWords", encoded) } + _wakeWords.value = sanitized + } + + fun setVoiceWakeMode(mode: VoiceWakeMode) { + prefs.edit { putString(voiceWakeModeKey, mode.rawValue) } + _voiceWakeMode.value = mode + } + + fun setTalkEnabled(value: Boolean) { + prefs.edit { putBoolean("talk.enabled", value) } + _talkEnabled.value = value + } + + private fun loadVoiceWakeMode(): VoiceWakeMode { + val raw = prefs.getString(voiceWakeModeKey, null) + val resolved = VoiceWakeMode.fromRawValue(raw) + + // Default ON (foreground) when unset. + if (raw.isNullOrBlank()) { + prefs.edit { putString(voiceWakeModeKey, resolved.rawValue) } + } + + return resolved + } + + private fun loadWakeWords(): List { + val raw = prefs.getString("voiceWake.triggerWords", null)?.trim() + if (raw.isNullOrEmpty()) return defaultWakeWords + return try { + val element = json.parseToJsonElement(raw) + val array = element as? JsonArray ?: return defaultWakeWords + val decoded = + array.mapNotNull { item -> + when (item) { + is JsonNull -> null + is JsonPrimitive -> item.content.trim().takeIf { it.isNotEmpty() } + else -> null + } + } + WakeWords.sanitize(decoded, defaultWakeWords) + } catch (_: Throwable) { + defaultWakeWords + } + } + +} diff --git a/apps/android/app/src/main/java/ai/openclaw/android/SessionKey.kt b/apps/android/app/src/main/java/ai/openclaw/android/SessionKey.kt new file mode 100644 index 0000000000000000000000000000000000000000..8148a17029e1467eb2cca3dd94703f41b1089a7c --- /dev/null +++ b/apps/android/app/src/main/java/ai/openclaw/android/SessionKey.kt @@ -0,0 +1,13 @@ +package ai.openclaw.android + +internal fun normalizeMainKey(raw: String?): String { + val trimmed = raw?.trim() + return if (!trimmed.isNullOrEmpty()) trimmed else "main" +} + +internal fun isCanonicalMainSessionKey(raw: String?): Boolean { + val trimmed = raw?.trim().orEmpty() + if (trimmed.isEmpty()) return false + if (trimmed == "global") return true + return trimmed.startsWith("agent:") +} diff --git a/apps/android/app/src/main/java/ai/openclaw/android/VoiceWakeMode.kt b/apps/android/app/src/main/java/ai/openclaw/android/VoiceWakeMode.kt new file mode 100644 index 0000000000000000000000000000000000000000..75c2fe3446847a5be8c1d1759b03dfb61529a400 --- /dev/null +++ b/apps/android/app/src/main/java/ai/openclaw/android/VoiceWakeMode.kt @@ -0,0 +1,14 @@ +package ai.openclaw.android + +enum class VoiceWakeMode(val rawValue: String) { + Off("off"), + Foreground("foreground"), + Always("always"), + ; + + companion object { + fun fromRawValue(raw: String?): VoiceWakeMode { + return entries.firstOrNull { it.rawValue == raw?.trim()?.lowercase() } ?: Foreground + } + } +} diff --git a/apps/android/app/src/main/java/ai/openclaw/android/WakeWords.kt b/apps/android/app/src/main/java/ai/openclaw/android/WakeWords.kt new file mode 100644 index 0000000000000000000000000000000000000000..b64cb1dd749fe23655e9a3d075efdfacf88f01b6 --- /dev/null +++ b/apps/android/app/src/main/java/ai/openclaw/android/WakeWords.kt @@ -0,0 +1,21 @@ +package ai.openclaw.android + +object WakeWords { + const val maxWords: Int = 32 + const val maxWordLength: Int = 64 + + fun parseCommaSeparated(input: String): List { + return input.split(",").map { it.trim() }.filter { it.isNotEmpty() } + } + + fun parseIfChanged(input: String, current: List): List? { + val parsed = parseCommaSeparated(input) + return if (parsed == current) null else parsed + } + + fun sanitize(words: List, defaults: List): List { + val cleaned = + words.map { it.trim() }.filter { it.isNotEmpty() }.take(maxWords).map { it.take(maxWordLength) } + return cleaned.ifEmpty { defaults } + } +} diff --git a/apps/android/app/src/main/java/ai/openclaw/android/chat/ChatController.kt b/apps/android/app/src/main/java/ai/openclaw/android/chat/ChatController.kt new file mode 100644 index 0000000000000000000000000000000000000000..3ed69ee5b24316dd94f6f775bc3bead2e116925a --- /dev/null +++ b/apps/android/app/src/main/java/ai/openclaw/android/chat/ChatController.kt @@ -0,0 +1,524 @@ +package ai.openclaw.android.chat + +import ai.openclaw.android.gateway.GatewaySession +import java.util.UUID +import java.util.concurrent.ConcurrentHashMap +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Job +import kotlinx.coroutines.delay +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.asStateFlow +import kotlinx.coroutines.launch +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.JsonArray +import kotlinx.serialization.json.JsonElement +import kotlinx.serialization.json.JsonNull +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.JsonPrimitive +import kotlinx.serialization.json.buildJsonObject + +class ChatController( + private val scope: CoroutineScope, + private val session: GatewaySession, + private val json: Json, + private val supportsChatSubscribe: Boolean, +) { + private val _sessionKey = MutableStateFlow("main") + val sessionKey: StateFlow = _sessionKey.asStateFlow() + + private val _sessionId = MutableStateFlow(null) + val sessionId: StateFlow = _sessionId.asStateFlow() + + private val _messages = MutableStateFlow>(emptyList()) + val messages: StateFlow> = _messages.asStateFlow() + + private val _errorText = MutableStateFlow(null) + val errorText: StateFlow = _errorText.asStateFlow() + + private val _healthOk = MutableStateFlow(false) + val healthOk: StateFlow = _healthOk.asStateFlow() + + private val _thinkingLevel = MutableStateFlow("off") + val thinkingLevel: StateFlow = _thinkingLevel.asStateFlow() + + private val _pendingRunCount = MutableStateFlow(0) + val pendingRunCount: StateFlow = _pendingRunCount.asStateFlow() + + private val _streamingAssistantText = MutableStateFlow(null) + val streamingAssistantText: StateFlow = _streamingAssistantText.asStateFlow() + + private val pendingToolCallsById = ConcurrentHashMap() + private val _pendingToolCalls = MutableStateFlow>(emptyList()) + val pendingToolCalls: StateFlow> = _pendingToolCalls.asStateFlow() + + private val _sessions = MutableStateFlow>(emptyList()) + val sessions: StateFlow> = _sessions.asStateFlow() + + private val pendingRuns = mutableSetOf() + private val pendingRunTimeoutJobs = ConcurrentHashMap() + private val pendingRunTimeoutMs = 120_000L + + private var lastHealthPollAtMs: Long? = null + + fun onDisconnected(message: String) { + _healthOk.value = false + // Not an error; keep connection status in the UI pill. + _errorText.value = null + clearPendingRuns() + pendingToolCallsById.clear() + publishPendingToolCalls() + _streamingAssistantText.value = null + _sessionId.value = null + } + + fun load(sessionKey: String) { + val key = sessionKey.trim().ifEmpty { "main" } + _sessionKey.value = key + scope.launch { bootstrap(forceHealth = true) } + } + + fun applyMainSessionKey(mainSessionKey: String) { + val trimmed = mainSessionKey.trim() + if (trimmed.isEmpty()) return + if (_sessionKey.value == trimmed) return + if (_sessionKey.value != "main") return + _sessionKey.value = trimmed + scope.launch { bootstrap(forceHealth = true) } + } + + fun refresh() { + scope.launch { bootstrap(forceHealth = true) } + } + + fun refreshSessions(limit: Int? = null) { + scope.launch { fetchSessions(limit = limit) } + } + + fun setThinkingLevel(thinkingLevel: String) { + val normalized = normalizeThinking(thinkingLevel) + if (normalized == _thinkingLevel.value) return + _thinkingLevel.value = normalized + } + + fun switchSession(sessionKey: String) { + val key = sessionKey.trim() + if (key.isEmpty()) return + if (key == _sessionKey.value) return + _sessionKey.value = key + scope.launch { bootstrap(forceHealth = true) } + } + + fun sendMessage( + message: String, + thinkingLevel: String, + attachments: List, + ) { + val trimmed = message.trim() + if (trimmed.isEmpty() && attachments.isEmpty()) return + if (!_healthOk.value) { + _errorText.value = "Gateway health not OK; cannot send" + return + } + + val runId = UUID.randomUUID().toString() + val text = if (trimmed.isEmpty() && attachments.isNotEmpty()) "See attached." else trimmed + val sessionKey = _sessionKey.value + val thinking = normalizeThinking(thinkingLevel) + + // Optimistic user message. + val userContent = + buildList { + add(ChatMessageContent(type = "text", text = text)) + for (att in attachments) { + add( + ChatMessageContent( + type = att.type, + mimeType = att.mimeType, + fileName = att.fileName, + base64 = att.base64, + ), + ) + } + } + _messages.value = + _messages.value + + ChatMessage( + id = UUID.randomUUID().toString(), + role = "user", + content = userContent, + timestampMs = System.currentTimeMillis(), + ) + + armPendingRunTimeout(runId) + synchronized(pendingRuns) { + pendingRuns.add(runId) + _pendingRunCount.value = pendingRuns.size + } + + _errorText.value = null + _streamingAssistantText.value = null + pendingToolCallsById.clear() + publishPendingToolCalls() + + scope.launch { + try { + val params = + buildJsonObject { + put("sessionKey", JsonPrimitive(sessionKey)) + put("message", JsonPrimitive(text)) + put("thinking", JsonPrimitive(thinking)) + put("timeoutMs", JsonPrimitive(30_000)) + put("idempotencyKey", JsonPrimitive(runId)) + if (attachments.isNotEmpty()) { + put( + "attachments", + JsonArray( + attachments.map { att -> + buildJsonObject { + put("type", JsonPrimitive(att.type)) + put("mimeType", JsonPrimitive(att.mimeType)) + put("fileName", JsonPrimitive(att.fileName)) + put("content", JsonPrimitive(att.base64)) + } + }, + ), + ) + } + } + val res = session.request("chat.send", params.toString()) + val actualRunId = parseRunId(res) ?: runId + if (actualRunId != runId) { + clearPendingRun(runId) + armPendingRunTimeout(actualRunId) + synchronized(pendingRuns) { + pendingRuns.add(actualRunId) + _pendingRunCount.value = pendingRuns.size + } + } + } catch (err: Throwable) { + clearPendingRun(runId) + _errorText.value = err.message + } + } + } + + fun abort() { + val runIds = + synchronized(pendingRuns) { + pendingRuns.toList() + } + if (runIds.isEmpty()) return + scope.launch { + for (runId in runIds) { + try { + val params = + buildJsonObject { + put("sessionKey", JsonPrimitive(_sessionKey.value)) + put("runId", JsonPrimitive(runId)) + } + session.request("chat.abort", params.toString()) + } catch (_: Throwable) { + // best-effort + } + } + } + } + + fun handleGatewayEvent(event: String, payloadJson: String?) { + when (event) { + "tick" -> { + scope.launch { pollHealthIfNeeded(force = false) } + } + "health" -> { + // If we receive a health snapshot, the gateway is reachable. + _healthOk.value = true + } + "seqGap" -> { + _errorText.value = "Event stream interrupted; try refreshing." + clearPendingRuns() + } + "chat" -> { + if (payloadJson.isNullOrBlank()) return + handleChatEvent(payloadJson) + } + "agent" -> { + if (payloadJson.isNullOrBlank()) return + handleAgentEvent(payloadJson) + } + } + } + + private suspend fun bootstrap(forceHealth: Boolean) { + _errorText.value = null + _healthOk.value = false + clearPendingRuns() + pendingToolCallsById.clear() + publishPendingToolCalls() + _streamingAssistantText.value = null + _sessionId.value = null + + val key = _sessionKey.value + try { + if (supportsChatSubscribe) { + try { + session.sendNodeEvent("chat.subscribe", """{"sessionKey":"$key"}""") + } catch (_: Throwable) { + // best-effort + } + } + + val historyJson = session.request("chat.history", """{"sessionKey":"$key"}""") + val history = parseHistory(historyJson, sessionKey = key) + _messages.value = history.messages + _sessionId.value = history.sessionId + history.thinkingLevel?.trim()?.takeIf { it.isNotEmpty() }?.let { _thinkingLevel.value = it } + + pollHealthIfNeeded(force = forceHealth) + fetchSessions(limit = 50) + } catch (err: Throwable) { + _errorText.value = err.message + } + } + + private suspend fun fetchSessions(limit: Int?) { + try { + val params = + buildJsonObject { + put("includeGlobal", JsonPrimitive(true)) + put("includeUnknown", JsonPrimitive(false)) + if (limit != null && limit > 0) put("limit", JsonPrimitive(limit)) + } + val res = session.request("sessions.list", params.toString()) + _sessions.value = parseSessions(res) + } catch (_: Throwable) { + // best-effort + } + } + + private suspend fun pollHealthIfNeeded(force: Boolean) { + val now = System.currentTimeMillis() + val last = lastHealthPollAtMs + if (!force && last != null && now - last < 10_000) return + lastHealthPollAtMs = now + try { + session.request("health", null) + _healthOk.value = true + } catch (_: Throwable) { + _healthOk.value = false + } + } + + private fun handleChatEvent(payloadJson: String) { + val payload = json.parseToJsonElement(payloadJson).asObjectOrNull() ?: return + val sessionKey = payload["sessionKey"].asStringOrNull()?.trim() + if (!sessionKey.isNullOrEmpty() && sessionKey != _sessionKey.value) return + + val runId = payload["runId"].asStringOrNull() + if (runId != null) { + val isPending = + synchronized(pendingRuns) { + pendingRuns.contains(runId) + } + if (!isPending) return + } + + val state = payload["state"].asStringOrNull() + when (state) { + "final", "aborted", "error" -> { + if (state == "error") { + _errorText.value = payload["errorMessage"].asStringOrNull() ?: "Chat failed" + } + if (runId != null) clearPendingRun(runId) else clearPendingRuns() + pendingToolCallsById.clear() + publishPendingToolCalls() + _streamingAssistantText.value = null + scope.launch { + try { + val historyJson = + session.request("chat.history", """{"sessionKey":"${_sessionKey.value}"}""") + val history = parseHistory(historyJson, sessionKey = _sessionKey.value) + _messages.value = history.messages + _sessionId.value = history.sessionId + history.thinkingLevel?.trim()?.takeIf { it.isNotEmpty() }?.let { _thinkingLevel.value = it } + } catch (_: Throwable) { + // best-effort + } + } + } + } + } + + private fun handleAgentEvent(payloadJson: String) { + val payload = json.parseToJsonElement(payloadJson).asObjectOrNull() ?: return + val runId = payload["runId"].asStringOrNull() + val sessionId = _sessionId.value + if (sessionId != null && runId != sessionId) return + + val stream = payload["stream"].asStringOrNull() + val data = payload["data"].asObjectOrNull() + + when (stream) { + "assistant" -> { + val text = data?.get("text")?.asStringOrNull() + if (!text.isNullOrEmpty()) { + _streamingAssistantText.value = text + } + } + "tool" -> { + val phase = data?.get("phase")?.asStringOrNull() + val name = data?.get("name")?.asStringOrNull() + val toolCallId = data?.get("toolCallId")?.asStringOrNull() + if (phase.isNullOrEmpty() || name.isNullOrEmpty() || toolCallId.isNullOrEmpty()) return + + val ts = payload["ts"].asLongOrNull() ?: System.currentTimeMillis() + if (phase == "start") { + val args = data?.get("args").asObjectOrNull() + pendingToolCallsById[toolCallId] = + ChatPendingToolCall( + toolCallId = toolCallId, + name = name, + args = args, + startedAtMs = ts, + isError = null, + ) + publishPendingToolCalls() + } else if (phase == "result") { + pendingToolCallsById.remove(toolCallId) + publishPendingToolCalls() + } + } + "error" -> { + _errorText.value = "Event stream interrupted; try refreshing." + clearPendingRuns() + pendingToolCallsById.clear() + publishPendingToolCalls() + _streamingAssistantText.value = null + } + } + } + + private fun publishPendingToolCalls() { + _pendingToolCalls.value = + pendingToolCallsById.values.sortedBy { it.startedAtMs } + } + + private fun armPendingRunTimeout(runId: String) { + pendingRunTimeoutJobs[runId]?.cancel() + pendingRunTimeoutJobs[runId] = + scope.launch { + delay(pendingRunTimeoutMs) + val stillPending = + synchronized(pendingRuns) { + pendingRuns.contains(runId) + } + if (!stillPending) return@launch + clearPendingRun(runId) + _errorText.value = "Timed out waiting for a reply; try again or refresh." + } + } + + private fun clearPendingRun(runId: String) { + pendingRunTimeoutJobs.remove(runId)?.cancel() + synchronized(pendingRuns) { + pendingRuns.remove(runId) + _pendingRunCount.value = pendingRuns.size + } + } + + private fun clearPendingRuns() { + for ((_, job) in pendingRunTimeoutJobs) { + job.cancel() + } + pendingRunTimeoutJobs.clear() + synchronized(pendingRuns) { + pendingRuns.clear() + _pendingRunCount.value = 0 + } + } + + private fun parseHistory(historyJson: String, sessionKey: String): ChatHistory { + val root = json.parseToJsonElement(historyJson).asObjectOrNull() ?: return ChatHistory(sessionKey, null, null, emptyList()) + val sid = root["sessionId"].asStringOrNull() + val thinkingLevel = root["thinkingLevel"].asStringOrNull() + val array = root["messages"].asArrayOrNull() ?: JsonArray(emptyList()) + + val messages = + array.mapNotNull { item -> + val obj = item.asObjectOrNull() ?: return@mapNotNull null + val role = obj["role"].asStringOrNull() ?: return@mapNotNull null + val content = obj["content"].asArrayOrNull()?.mapNotNull(::parseMessageContent) ?: emptyList() + val ts = obj["timestamp"].asLongOrNull() + ChatMessage( + id = UUID.randomUUID().toString(), + role = role, + content = content, + timestampMs = ts, + ) + } + + return ChatHistory(sessionKey = sessionKey, sessionId = sid, thinkingLevel = thinkingLevel, messages = messages) + } + + private fun parseMessageContent(el: JsonElement): ChatMessageContent? { + val obj = el.asObjectOrNull() ?: return null + val type = obj["type"].asStringOrNull() ?: "text" + return if (type == "text") { + ChatMessageContent(type = "text", text = obj["text"].asStringOrNull()) + } else { + ChatMessageContent( + type = type, + mimeType = obj["mimeType"].asStringOrNull(), + fileName = obj["fileName"].asStringOrNull(), + base64 = obj["content"].asStringOrNull(), + ) + } + } + + private fun parseSessions(jsonString: String): List { + val root = json.parseToJsonElement(jsonString).asObjectOrNull() ?: return emptyList() + val sessions = root["sessions"].asArrayOrNull() ?: return emptyList() + return sessions.mapNotNull { item -> + val obj = item.asObjectOrNull() ?: return@mapNotNull null + val key = obj["key"].asStringOrNull()?.trim().orEmpty() + if (key.isEmpty()) return@mapNotNull null + val updatedAt = obj["updatedAt"].asLongOrNull() + val displayName = obj["displayName"].asStringOrNull()?.trim() + ChatSessionEntry(key = key, updatedAtMs = updatedAt, displayName = displayName) + } + } + + private fun parseRunId(resJson: String): String? { + return try { + json.parseToJsonElement(resJson).asObjectOrNull()?.get("runId").asStringOrNull() + } catch (_: Throwable) { + null + } + } + + private fun normalizeThinking(raw: String): String { + return when (raw.trim().lowercase()) { + "low" -> "low" + "medium" -> "medium" + "high" -> "high" + else -> "off" + } + } +} + +private fun JsonElement?.asObjectOrNull(): JsonObject? = this as? JsonObject + +private fun JsonElement?.asArrayOrNull(): JsonArray? = this as? JsonArray + +private fun JsonElement?.asStringOrNull(): String? = + when (this) { + is JsonNull -> null + is JsonPrimitive -> content + else -> null + } + +private fun JsonElement?.asLongOrNull(): Long? = + when (this) { + is JsonPrimitive -> content.toLongOrNull() + else -> null + } diff --git a/apps/android/app/src/main/java/ai/openclaw/android/chat/ChatModels.kt b/apps/android/app/src/main/java/ai/openclaw/android/chat/ChatModels.kt new file mode 100644 index 0000000000000000000000000000000000000000..dd17a8c1ae56ea05584067994bb739f2e0a5d312 --- /dev/null +++ b/apps/android/app/src/main/java/ai/openclaw/android/chat/ChatModels.kt @@ -0,0 +1,44 @@ +package ai.openclaw.android.chat + +data class ChatMessage( + val id: String, + val role: String, + val content: List, + val timestampMs: Long?, +) + +data class ChatMessageContent( + val type: String = "text", + val text: String? = null, + val mimeType: String? = null, + val fileName: String? = null, + val base64: String? = null, +) + +data class ChatPendingToolCall( + val toolCallId: String, + val name: String, + val args: kotlinx.serialization.json.JsonObject? = null, + val startedAtMs: Long, + val isError: Boolean? = null, +) + +data class ChatSessionEntry( + val key: String, + val updatedAtMs: Long?, + val displayName: String? = null, +) + +data class ChatHistory( + val sessionKey: String, + val sessionId: String?, + val thinkingLevel: String?, + val messages: List, +) + +data class OutgoingAttachment( + val type: String, + val mimeType: String, + val fileName: String, + val base64: String, +) diff --git a/apps/android/app/src/main/java/ai/openclaw/android/gateway/BonjourEscapes.kt b/apps/android/app/src/main/java/ai/openclaw/android/gateway/BonjourEscapes.kt new file mode 100644 index 0000000000000000000000000000000000000000..1606df79ec6da5580345b28899e7474071827ecf --- /dev/null +++ b/apps/android/app/src/main/java/ai/openclaw/android/gateway/BonjourEscapes.kt @@ -0,0 +1,35 @@ +package ai.openclaw.android.gateway + +object BonjourEscapes { + fun decode(input: String): String { + if (input.isEmpty()) return input + + val bytes = mutableListOf() + var i = 0 + while (i < input.length) { + if (input[i] == '\\' && i + 3 < input.length) { + val d0 = input[i + 1] + val d1 = input[i + 2] + val d2 = input[i + 3] + if (d0.isDigit() && d1.isDigit() && d2.isDigit()) { + val value = + ((d0.code - '0'.code) * 100) + ((d1.code - '0'.code) * 10) + (d2.code - '0'.code) + if (value in 0..255) { + bytes.add(value.toByte()) + i += 4 + continue + } + } + } + + val codePoint = Character.codePointAt(input, i) + val charBytes = String(Character.toChars(codePoint)).toByteArray(Charsets.UTF_8) + for (b in charBytes) { + bytes.add(b) + } + i += Character.charCount(codePoint) + } + + return String(bytes.toByteArray(), Charsets.UTF_8) + } +} diff --git a/apps/android/app/src/main/java/ai/openclaw/android/gateway/DeviceAuthStore.kt b/apps/android/app/src/main/java/ai/openclaw/android/gateway/DeviceAuthStore.kt new file mode 100644 index 0000000000000000000000000000000000000000..810e029fba891b3269f531fbf3df96e593b3f605 --- /dev/null +++ b/apps/android/app/src/main/java/ai/openclaw/android/gateway/DeviceAuthStore.kt @@ -0,0 +1,26 @@ +package ai.openclaw.android.gateway + +import ai.openclaw.android.SecurePrefs + +class DeviceAuthStore(private val prefs: SecurePrefs) { + fun loadToken(deviceId: String, role: String): String? { + val key = tokenKey(deviceId, role) + return prefs.getString(key)?.trim()?.takeIf { it.isNotEmpty() } + } + + fun saveToken(deviceId: String, role: String, token: String) { + val key = tokenKey(deviceId, role) + prefs.putString(key, token.trim()) + } + + fun clearToken(deviceId: String, role: String) { + val key = tokenKey(deviceId, role) + prefs.remove(key) + } + + private fun tokenKey(deviceId: String, role: String): String { + val normalizedDevice = deviceId.trim().lowercase() + val normalizedRole = role.trim().lowercase() + return "gateway.deviceToken.$normalizedDevice.$normalizedRole" + } +} diff --git a/apps/android/app/src/main/java/ai/openclaw/android/gateway/DeviceIdentityStore.kt b/apps/android/app/src/main/java/ai/openclaw/android/gateway/DeviceIdentityStore.kt new file mode 100644 index 0000000000000000000000000000000000000000..accbb79e4dd6b981c437db385a013076b9471df8 --- /dev/null +++ b/apps/android/app/src/main/java/ai/openclaw/android/gateway/DeviceIdentityStore.kt @@ -0,0 +1,150 @@ +package ai.openclaw.android.gateway + +import android.content.Context +import android.util.Base64 +import java.io.File +import java.security.KeyFactory +import java.security.KeyPairGenerator +import java.security.MessageDigest +import java.security.Signature +import java.security.spec.PKCS8EncodedKeySpec +import kotlinx.serialization.Serializable +import kotlinx.serialization.json.Json + +@Serializable +data class DeviceIdentity( + val deviceId: String, + val publicKeyRawBase64: String, + val privateKeyPkcs8Base64: String, + val createdAtMs: Long, +) + +class DeviceIdentityStore(context: Context) { + private val json = Json { ignoreUnknownKeys = true } + private val identityFile = File(context.filesDir, "openclaw/identity/device.json") + + @Synchronized + fun loadOrCreate(): DeviceIdentity { + val existing = load() + if (existing != null) { + val derived = deriveDeviceId(existing.publicKeyRawBase64) + if (derived != null && derived != existing.deviceId) { + val updated = existing.copy(deviceId = derived) + save(updated) + return updated + } + return existing + } + val fresh = generate() + save(fresh) + return fresh + } + + fun signPayload(payload: String, identity: DeviceIdentity): String? { + return try { + val privateKeyBytes = Base64.decode(identity.privateKeyPkcs8Base64, Base64.DEFAULT) + val keySpec = PKCS8EncodedKeySpec(privateKeyBytes) + val keyFactory = KeyFactory.getInstance("Ed25519") + val privateKey = keyFactory.generatePrivate(keySpec) + val signature = Signature.getInstance("Ed25519") + signature.initSign(privateKey) + signature.update(payload.toByteArray(Charsets.UTF_8)) + base64UrlEncode(signature.sign()) + } catch (_: Throwable) { + null + } + } + + fun publicKeyBase64Url(identity: DeviceIdentity): String? { + return try { + val raw = Base64.decode(identity.publicKeyRawBase64, Base64.DEFAULT) + base64UrlEncode(raw) + } catch (_: Throwable) { + null + } + } + + private fun load(): DeviceIdentity? { + return readIdentity(identityFile) + } + + private fun readIdentity(file: File): DeviceIdentity? { + return try { + if (!file.exists()) return null + val raw = file.readText(Charsets.UTF_8) + val decoded = json.decodeFromString(DeviceIdentity.serializer(), raw) + if (decoded.deviceId.isBlank() || + decoded.publicKeyRawBase64.isBlank() || + decoded.privateKeyPkcs8Base64.isBlank() + ) { + null + } else { + decoded + } + } catch (_: Throwable) { + null + } + } + + private fun save(identity: DeviceIdentity) { + try { + identityFile.parentFile?.mkdirs() + val encoded = json.encodeToString(DeviceIdentity.serializer(), identity) + identityFile.writeText(encoded, Charsets.UTF_8) + } catch (_: Throwable) { + // best-effort only + } + } + + private fun generate(): DeviceIdentity { + val keyPair = KeyPairGenerator.getInstance("Ed25519").generateKeyPair() + val spki = keyPair.public.encoded + val rawPublic = stripSpkiPrefix(spki) + val deviceId = sha256Hex(rawPublic) + val privateKey = keyPair.private.encoded + return DeviceIdentity( + deviceId = deviceId, + publicKeyRawBase64 = Base64.encodeToString(rawPublic, Base64.NO_WRAP), + privateKeyPkcs8Base64 = Base64.encodeToString(privateKey, Base64.NO_WRAP), + createdAtMs = System.currentTimeMillis(), + ) + } + + private fun deriveDeviceId(publicKeyRawBase64: String): String? { + return try { + val raw = Base64.decode(publicKeyRawBase64, Base64.DEFAULT) + sha256Hex(raw) + } catch (_: Throwable) { + null + } + } + + private fun stripSpkiPrefix(spki: ByteArray): ByteArray { + if (spki.size == ED25519_SPKI_PREFIX.size + 32 && + spki.copyOfRange(0, ED25519_SPKI_PREFIX.size).contentEquals(ED25519_SPKI_PREFIX) + ) { + return spki.copyOfRange(ED25519_SPKI_PREFIX.size, spki.size) + } + return spki + } + + private fun sha256Hex(data: ByteArray): String { + val digest = MessageDigest.getInstance("SHA-256").digest(data) + val out = StringBuilder(digest.size * 2) + for (byte in digest) { + out.append(String.format("%02x", byte)) + } + return out.toString() + } + + private fun base64UrlEncode(data: ByteArray): String { + return Base64.encodeToString(data, Base64.URL_SAFE or Base64.NO_WRAP or Base64.NO_PADDING) + } + + companion object { + private val ED25519_SPKI_PREFIX = + byteArrayOf( + 0x30, 0x2a, 0x30, 0x05, 0x06, 0x03, 0x2b, 0x65, 0x70, 0x03, 0x21, 0x00, + ) + } +} diff --git a/apps/android/app/src/main/java/ai/openclaw/android/gateway/GatewayDiscovery.kt b/apps/android/app/src/main/java/ai/openclaw/android/gateway/GatewayDiscovery.kt new file mode 100644 index 0000000000000000000000000000000000000000..2ad8ec0cb195a5455c61a7e20def2cf8cff6af6c --- /dev/null +++ b/apps/android/app/src/main/java/ai/openclaw/android/gateway/GatewayDiscovery.kt @@ -0,0 +1,521 @@ +package ai.openclaw.android.gateway + +import android.content.Context +import android.net.ConnectivityManager +import android.net.DnsResolver +import android.net.NetworkCapabilities +import android.net.nsd.NsdManager +import android.net.nsd.NsdServiceInfo +import android.os.CancellationSignal +import android.util.Log +import java.io.IOException +import java.net.InetSocketAddress +import java.nio.ByteBuffer +import java.nio.charset.CodingErrorAction +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.Executor +import java.util.concurrent.Executors +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.Job +import kotlinx.coroutines.delay +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.asStateFlow +import kotlinx.coroutines.launch +import kotlinx.coroutines.suspendCancellableCoroutine +import org.xbill.DNS.AAAARecord +import org.xbill.DNS.ARecord +import org.xbill.DNS.DClass +import org.xbill.DNS.ExtendedResolver +import org.xbill.DNS.Message +import org.xbill.DNS.Name +import org.xbill.DNS.PTRRecord +import org.xbill.DNS.Record +import org.xbill.DNS.Rcode +import org.xbill.DNS.Resolver +import org.xbill.DNS.SRVRecord +import org.xbill.DNS.Section +import org.xbill.DNS.SimpleResolver +import org.xbill.DNS.TextParseException +import org.xbill.DNS.TXTRecord +import org.xbill.DNS.Type +import kotlin.coroutines.resume +import kotlin.coroutines.resumeWithException + +@Suppress("DEPRECATION") +class GatewayDiscovery( + context: Context, + private val scope: CoroutineScope, +) { + private val nsd = context.getSystemService(NsdManager::class.java) + private val connectivity = context.getSystemService(ConnectivityManager::class.java) + private val dns = DnsResolver.getInstance() + private val serviceType = "_openclaw-gw._tcp." + private val wideAreaDomain = System.getenv("OPENCLAW_WIDE_AREA_DOMAIN") + private val logTag = "OpenClaw/GatewayDiscovery" + + private val localById = ConcurrentHashMap() + private val unicastById = ConcurrentHashMap() + private val _gateways = MutableStateFlow>(emptyList()) + val gateways: StateFlow> = _gateways.asStateFlow() + + private val _statusText = MutableStateFlow("Searching…") + val statusText: StateFlow = _statusText.asStateFlow() + + private var unicastJob: Job? = null + private val dnsExecutor: Executor = Executors.newCachedThreadPool() + + @Volatile private var lastWideAreaRcode: Int? = null + @Volatile private var lastWideAreaCount: Int = 0 + + private val discoveryListener = + object : NsdManager.DiscoveryListener { + override fun onStartDiscoveryFailed(serviceType: String, errorCode: Int) {} + override fun onStopDiscoveryFailed(serviceType: String, errorCode: Int) {} + override fun onDiscoveryStarted(serviceType: String) {} + override fun onDiscoveryStopped(serviceType: String) {} + + override fun onServiceFound(serviceInfo: NsdServiceInfo) { + if (serviceInfo.serviceType != this@GatewayDiscovery.serviceType) return + resolve(serviceInfo) + } + + override fun onServiceLost(serviceInfo: NsdServiceInfo) { + val serviceName = BonjourEscapes.decode(serviceInfo.serviceName) + val id = stableId(serviceName, "local.") + localById.remove(id) + publish() + } + } + + init { + startLocalDiscovery() + if (!wideAreaDomain.isNullOrBlank()) { + startUnicastDiscovery(wideAreaDomain) + } + } + + private fun startLocalDiscovery() { + try { + nsd.discoverServices(serviceType, NsdManager.PROTOCOL_DNS_SD, discoveryListener) + } catch (_: Throwable) { + // ignore (best-effort) + } + } + + private fun stopLocalDiscovery() { + try { + nsd.stopServiceDiscovery(discoveryListener) + } catch (_: Throwable) { + // ignore (best-effort) + } + } + + private fun startUnicastDiscovery(domain: String) { + unicastJob = + scope.launch(Dispatchers.IO) { + while (true) { + try { + refreshUnicast(domain) + } catch (_: Throwable) { + // ignore (best-effort) + } + delay(5000) + } + } + } + + private fun resolve(serviceInfo: NsdServiceInfo) { + nsd.resolveService( + serviceInfo, + object : NsdManager.ResolveListener { + override fun onResolveFailed(serviceInfo: NsdServiceInfo, errorCode: Int) {} + + override fun onServiceResolved(resolved: NsdServiceInfo) { + val host = resolved.host?.hostAddress ?: return + val port = resolved.port + if (port <= 0) return + + val rawServiceName = resolved.serviceName + val serviceName = BonjourEscapes.decode(rawServiceName) + val displayName = BonjourEscapes.decode(txt(resolved, "displayName") ?: serviceName) + val lanHost = txt(resolved, "lanHost") + val tailnetDns = txt(resolved, "tailnetDns") + val gatewayPort = txtInt(resolved, "gatewayPort") + val canvasPort = txtInt(resolved, "canvasPort") + val tlsEnabled = txtBool(resolved, "gatewayTls") + val tlsFingerprint = txt(resolved, "gatewayTlsSha256") + val id = stableId(serviceName, "local.") + localById[id] = + GatewayEndpoint( + stableId = id, + name = displayName, + host = host, + port = port, + lanHost = lanHost, + tailnetDns = tailnetDns, + gatewayPort = gatewayPort, + canvasPort = canvasPort, + tlsEnabled = tlsEnabled, + tlsFingerprintSha256 = tlsFingerprint, + ) + publish() + } + }, + ) + } + + private fun publish() { + _gateways.value = + (localById.values + unicastById.values).sortedBy { it.name.lowercase() } + _statusText.value = buildStatusText() + } + + private fun buildStatusText(): String { + val localCount = localById.size + val wideRcode = lastWideAreaRcode + val wideCount = lastWideAreaCount + + val wide = + when (wideRcode) { + null -> "Wide: ?" + Rcode.NOERROR -> "Wide: $wideCount" + Rcode.NXDOMAIN -> "Wide: NXDOMAIN" + else -> "Wide: ${Rcode.string(wideRcode)}" + } + + return when { + localCount == 0 && wideRcode == null -> "Searching for gateways…" + localCount == 0 -> "$wide" + else -> "Local: $localCount • $wide" + } + } + + private fun stableId(serviceName: String, domain: String): String { + return "${serviceType}|${domain}|${normalizeName(serviceName)}" + } + + private fun normalizeName(raw: String): String { + return raw.trim().split(Regex("\\s+")).joinToString(" ") + } + + private fun txt(info: NsdServiceInfo, key: String): String? { + val bytes = info.attributes[key] ?: return null + return try { + String(bytes, Charsets.UTF_8).trim().ifEmpty { null } + } catch (_: Throwable) { + null + } + } + + private fun txtInt(info: NsdServiceInfo, key: String): Int? { + return txt(info, key)?.toIntOrNull() + } + + private fun txtBool(info: NsdServiceInfo, key: String): Boolean { + val raw = txt(info, key)?.trim()?.lowercase() ?: return false + return raw == "1" || raw == "true" || raw == "yes" + } + + private suspend fun refreshUnicast(domain: String) { + val ptrName = "${serviceType}${domain}" + val ptrMsg = lookupUnicastMessage(ptrName, Type.PTR) ?: return + val ptrRecords = records(ptrMsg, Section.ANSWER).mapNotNull { it as? PTRRecord } + + val next = LinkedHashMap() + for (ptr in ptrRecords) { + val instanceFqdn = ptr.target.toString() + val srv = + recordByName(ptrMsg, instanceFqdn, Type.SRV) as? SRVRecord + ?: run { + val msg = lookupUnicastMessage(instanceFqdn, Type.SRV) ?: return@run null + recordByName(msg, instanceFqdn, Type.SRV) as? SRVRecord + } + ?: continue + val port = srv.port + if (port <= 0) continue + + val targetFqdn = srv.target.toString() + val host = + resolveHostFromMessage(ptrMsg, targetFqdn) + ?: resolveHostFromMessage(lookupUnicastMessage(instanceFqdn, Type.SRV), targetFqdn) + ?: resolveHostUnicast(targetFqdn) + ?: continue + + val txtFromPtr = + recordsByName(ptrMsg, Section.ADDITIONAL)[keyName(instanceFqdn)] + .orEmpty() + .mapNotNull { it as? TXTRecord } + val txt = + if (txtFromPtr.isNotEmpty()) { + txtFromPtr + } else { + val msg = lookupUnicastMessage(instanceFqdn, Type.TXT) + records(msg, Section.ANSWER).mapNotNull { it as? TXTRecord } + } + val instanceName = BonjourEscapes.decode(decodeInstanceName(instanceFqdn, domain)) + val displayName = BonjourEscapes.decode(txtValue(txt, "displayName") ?: instanceName) + val lanHost = txtValue(txt, "lanHost") + val tailnetDns = txtValue(txt, "tailnetDns") + val gatewayPort = txtIntValue(txt, "gatewayPort") + val canvasPort = txtIntValue(txt, "canvasPort") + val tlsEnabled = txtBoolValue(txt, "gatewayTls") + val tlsFingerprint = txtValue(txt, "gatewayTlsSha256") + val id = stableId(instanceName, domain) + next[id] = + GatewayEndpoint( + stableId = id, + name = displayName, + host = host, + port = port, + lanHost = lanHost, + tailnetDns = tailnetDns, + gatewayPort = gatewayPort, + canvasPort = canvasPort, + tlsEnabled = tlsEnabled, + tlsFingerprintSha256 = tlsFingerprint, + ) + } + + unicastById.clear() + unicastById.putAll(next) + lastWideAreaRcode = ptrMsg.header.rcode + lastWideAreaCount = next.size + publish() + + if (next.isEmpty()) { + Log.d( + logTag, + "wide-area discovery: 0 results for $ptrName (rcode=${Rcode.string(ptrMsg.header.rcode)})", + ) + } + } + + private fun decodeInstanceName(instanceFqdn: String, domain: String): String { + val suffix = "${serviceType}${domain}" + val withoutSuffix = + if (instanceFqdn.endsWith(suffix)) { + instanceFqdn.removeSuffix(suffix) + } else { + instanceFqdn.substringBefore(serviceType) + } + return normalizeName(stripTrailingDot(withoutSuffix)) + } + + private fun stripTrailingDot(raw: String): String { + return raw.removeSuffix(".") + } + + private suspend fun lookupUnicastMessage(name: String, type: Int): Message? { + val query = + try { + Message.newQuery( + org.xbill.DNS.Record.newRecord( + Name.fromString(name), + type, + DClass.IN, + ), + ) + } catch (_: TextParseException) { + return null + } + + val system = queryViaSystemDns(query) + if (records(system, Section.ANSWER).any { it.type == type }) return system + + val direct = createDirectResolver() ?: return system + return try { + val msg = direct.send(query) + if (records(msg, Section.ANSWER).any { it.type == type }) msg else system + } catch (_: Throwable) { + system + } + } + + private suspend fun queryViaSystemDns(query: Message): Message? { + val network = preferredDnsNetwork() + val bytes = + try { + rawQuery(network, query.toWire()) + } catch (_: Throwable) { + return null + } + + return try { + Message(bytes) + } catch (_: IOException) { + null + } + } + + private fun records(msg: Message?, section: Int): List { + return msg?.getSectionArray(section)?.toList() ?: emptyList() + } + + private fun keyName(raw: String): String { + return raw.trim().lowercase() + } + + private fun recordsByName(msg: Message, section: Int): Map> { + val next = LinkedHashMap>() + for (r in records(msg, section)) { + val name = r.name?.toString() ?: continue + next.getOrPut(keyName(name)) { mutableListOf() }.add(r) + } + return next + } + + private fun recordByName(msg: Message, fqdn: String, type: Int): Record? { + val key = keyName(fqdn) + val byNameAnswer = recordsByName(msg, Section.ANSWER) + val fromAnswer = byNameAnswer[key].orEmpty().firstOrNull { it.type == type } + if (fromAnswer != null) return fromAnswer + + val byNameAdditional = recordsByName(msg, Section.ADDITIONAL) + return byNameAdditional[key].orEmpty().firstOrNull { it.type == type } + } + + private fun resolveHostFromMessage(msg: Message?, hostname: String): String? { + val m = msg ?: return null + val key = keyName(hostname) + val additional = recordsByName(m, Section.ADDITIONAL)[key].orEmpty() + val a = additional.mapNotNull { it as? ARecord }.mapNotNull { it.address?.hostAddress } + val aaaa = additional.mapNotNull { it as? AAAARecord }.mapNotNull { it.address?.hostAddress } + return a.firstOrNull() ?: aaaa.firstOrNull() + } + + private fun preferredDnsNetwork(): android.net.Network? { + val cm = connectivity ?: return null + + // Prefer VPN (Tailscale) when present; otherwise use the active network. + cm.allNetworks.firstOrNull { n -> + val caps = cm.getNetworkCapabilities(n) ?: return@firstOrNull false + caps.hasTransport(NetworkCapabilities.TRANSPORT_VPN) + }?.let { return it } + + return cm.activeNetwork + } + + private fun createDirectResolver(): Resolver? { + val cm = connectivity ?: return null + + val candidateNetworks = + buildList { + cm.allNetworks + .firstOrNull { n -> + val caps = cm.getNetworkCapabilities(n) ?: return@firstOrNull false + caps.hasTransport(NetworkCapabilities.TRANSPORT_VPN) + }?.let(::add) + cm.activeNetwork?.let(::add) + }.distinct() + + val servers = + candidateNetworks + .asSequence() + .flatMap { n -> + cm.getLinkProperties(n)?.dnsServers?.asSequence() ?: emptySequence() + } + .distinctBy { it.hostAddress ?: it.toString() } + .toList() + if (servers.isEmpty()) return null + + return try { + val resolvers = + servers.mapNotNull { addr -> + try { + SimpleResolver().apply { + setAddress(InetSocketAddress(addr, 53)) + setTimeout(3) + } + } catch (_: Throwable) { + null + } + } + if (resolvers.isEmpty()) return null + ExtendedResolver(resolvers.toTypedArray()).apply { setTimeout(3) } + } catch (_: Throwable) { + null + } + } + + private suspend fun rawQuery(network: android.net.Network?, wireQuery: ByteArray): ByteArray = + suspendCancellableCoroutine { cont -> + val signal = CancellationSignal() + cont.invokeOnCancellation { signal.cancel() } + + dns.rawQuery( + network, + wireQuery, + DnsResolver.FLAG_EMPTY, + dnsExecutor, + signal, + object : DnsResolver.Callback { + override fun onAnswer(answer: ByteArray, rcode: Int) { + cont.resume(answer) + } + + override fun onError(error: DnsResolver.DnsException) { + cont.resumeWithException(error) + } + }, + ) + } + + private fun txtValue(records: List, key: String): String? { + val prefix = "$key=" + for (r in records) { + val strings: List = + try { + r.strings.mapNotNull { it as? String } + } catch (_: Throwable) { + emptyList() + } + for (s in strings) { + val trimmed = decodeDnsTxtString(s).trim() + if (trimmed.startsWith(prefix)) { + return trimmed.removePrefix(prefix).trim().ifEmpty { null } + } + } + } + return null + } + + private fun txtIntValue(records: List, key: String): Int? { + return txtValue(records, key)?.toIntOrNull() + } + + private fun txtBoolValue(records: List, key: String): Boolean { + val raw = txtValue(records, key)?.trim()?.lowercase() ?: return false + return raw == "1" || raw == "true" || raw == "yes" + } + + private fun decodeDnsTxtString(raw: String): String { + // dnsjava treats TXT as opaque bytes and decodes as ISO-8859-1 to preserve bytes. + // Our TXT payload is UTF-8 (written by the gateway), so re-decode when possible. + val bytes = raw.toByteArray(Charsets.ISO_8859_1) + val decoder = + Charsets.UTF_8 + .newDecoder() + .onMalformedInput(CodingErrorAction.REPORT) + .onUnmappableCharacter(CodingErrorAction.REPORT) + return try { + decoder.decode(ByteBuffer.wrap(bytes)).toString() + } catch (_: Throwable) { + raw + } + } + + private suspend fun resolveHostUnicast(hostname: String): String? { + val a = + records(lookupUnicastMessage(hostname, Type.A), Section.ANSWER) + .mapNotNull { it as? ARecord } + .mapNotNull { it.address?.hostAddress } + val aaaa = + records(lookupUnicastMessage(hostname, Type.AAAA), Section.ANSWER) + .mapNotNull { it as? AAAARecord } + .mapNotNull { it.address?.hostAddress } + + return a.firstOrNull() ?: aaaa.firstOrNull() + } +} diff --git a/apps/android/app/src/main/java/ai/openclaw/android/gateway/GatewayEndpoint.kt b/apps/android/app/src/main/java/ai/openclaw/android/gateway/GatewayEndpoint.kt new file mode 100644 index 0000000000000000000000000000000000000000..9a30106028297cb1fed0a7a937505e4ae7632ce5 --- /dev/null +++ b/apps/android/app/src/main/java/ai/openclaw/android/gateway/GatewayEndpoint.kt @@ -0,0 +1,26 @@ +package ai.openclaw.android.gateway + +data class GatewayEndpoint( + val stableId: String, + val name: String, + val host: String, + val port: Int, + val lanHost: String? = null, + val tailnetDns: String? = null, + val gatewayPort: Int? = null, + val canvasPort: Int? = null, + val tlsEnabled: Boolean = false, + val tlsFingerprintSha256: String? = null, +) { + companion object { + fun manual(host: String, port: Int): GatewayEndpoint = + GatewayEndpoint( + stableId = "manual|${host.lowercase()}|$port", + name = "$host:$port", + host = host, + port = port, + tlsEnabled = false, + tlsFingerprintSha256 = null, + ) + } +} diff --git a/apps/android/app/src/main/java/ai/openclaw/android/gateway/GatewayProtocol.kt b/apps/android/app/src/main/java/ai/openclaw/android/gateway/GatewayProtocol.kt new file mode 100644 index 0000000000000000000000000000000000000000..da8fa4c69330de9353f553fd3bb756b7bca829f4 --- /dev/null +++ b/apps/android/app/src/main/java/ai/openclaw/android/gateway/GatewayProtocol.kt @@ -0,0 +1,3 @@ +package ai.openclaw.android.gateway + +const val GATEWAY_PROTOCOL_VERSION = 3 diff --git a/apps/android/app/src/main/java/ai/openclaw/android/gateway/GatewaySession.kt b/apps/android/app/src/main/java/ai/openclaw/android/gateway/GatewaySession.kt new file mode 100644 index 0000000000000000000000000000000000000000..a8979d2e524700f567e0564084b6a64cb1868995 --- /dev/null +++ b/apps/android/app/src/main/java/ai/openclaw/android/gateway/GatewaySession.kt @@ -0,0 +1,683 @@ +package ai.openclaw.android.gateway + +import android.util.Log +import java.util.Locale +import java.util.UUID +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.atomic.AtomicBoolean +import kotlinx.coroutines.CompletableDeferred +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.Job +import kotlinx.coroutines.TimeoutCancellationException +import kotlinx.coroutines.cancelAndJoin +import kotlinx.coroutines.delay +import kotlinx.coroutines.isActive +import kotlinx.coroutines.launch +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock +import kotlinx.coroutines.withContext +import kotlinx.coroutines.withTimeout +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.JsonArray +import kotlinx.serialization.json.JsonElement +import kotlinx.serialization.json.JsonNull +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.JsonPrimitive +import kotlinx.serialization.json.buildJsonObject +import okhttp3.OkHttpClient +import okhttp3.Request +import okhttp3.Response +import okhttp3.WebSocket +import okhttp3.WebSocketListener + +data class GatewayClientInfo( + val id: String, + val displayName: String?, + val version: String, + val platform: String, + val mode: String, + val instanceId: String?, + val deviceFamily: String?, + val modelIdentifier: String?, +) + +data class GatewayConnectOptions( + val role: String, + val scopes: List, + val caps: List, + val commands: List, + val permissions: Map, + val client: GatewayClientInfo, + val userAgent: String? = null, +) + +class GatewaySession( + private val scope: CoroutineScope, + private val identityStore: DeviceIdentityStore, + private val deviceAuthStore: DeviceAuthStore, + private val onConnected: (serverName: String?, remoteAddress: String?, mainSessionKey: String?) -> Unit, + private val onDisconnected: (message: String) -> Unit, + private val onEvent: (event: String, payloadJson: String?) -> Unit, + private val onInvoke: (suspend (InvokeRequest) -> InvokeResult)? = null, + private val onTlsFingerprint: ((stableId: String, fingerprint: String) -> Unit)? = null, +) { + data class InvokeRequest( + val id: String, + val nodeId: String, + val command: String, + val paramsJson: String?, + val timeoutMs: Long?, + ) + + data class InvokeResult(val ok: Boolean, val payloadJson: String?, val error: ErrorShape?) { + companion object { + fun ok(payloadJson: String?) = InvokeResult(ok = true, payloadJson = payloadJson, error = null) + fun error(code: String, message: String) = + InvokeResult(ok = false, payloadJson = null, error = ErrorShape(code = code, message = message)) + } + } + + data class ErrorShape(val code: String, val message: String) + + private val json = Json { ignoreUnknownKeys = true } + private val writeLock = Mutex() + private val pending = ConcurrentHashMap>() + + @Volatile private var canvasHostUrl: String? = null + @Volatile private var mainSessionKey: String? = null + + private data class DesiredConnection( + val endpoint: GatewayEndpoint, + val token: String?, + val password: String?, + val options: GatewayConnectOptions, + val tls: GatewayTlsParams?, + ) + + private var desired: DesiredConnection? = null + private var job: Job? = null + @Volatile private var currentConnection: Connection? = null + + fun connect( + endpoint: GatewayEndpoint, + token: String?, + password: String?, + options: GatewayConnectOptions, + tls: GatewayTlsParams? = null, + ) { + desired = DesiredConnection(endpoint, token, password, options, tls) + if (job == null) { + job = scope.launch(Dispatchers.IO) { runLoop() } + } + } + + fun disconnect() { + desired = null + currentConnection?.closeQuietly() + scope.launch(Dispatchers.IO) { + job?.cancelAndJoin() + job = null + canvasHostUrl = null + mainSessionKey = null + onDisconnected("Offline") + } + } + + fun reconnect() { + currentConnection?.closeQuietly() + } + + fun currentCanvasHostUrl(): String? = canvasHostUrl + fun currentMainSessionKey(): String? = mainSessionKey + + suspend fun sendNodeEvent(event: String, payloadJson: String?) { + val conn = currentConnection ?: return + val parsedPayload = payloadJson?.let { parseJsonOrNull(it) } + val params = + buildJsonObject { + put("event", JsonPrimitive(event)) + if (parsedPayload != null) { + put("payload", parsedPayload) + } else if (payloadJson != null) { + put("payloadJSON", JsonPrimitive(payloadJson)) + } else { + put("payloadJSON", JsonNull) + } + } + try { + conn.request("node.event", params, timeoutMs = 8_000) + } catch (err: Throwable) { + Log.w("OpenClawGateway", "node.event failed: ${err.message ?: err::class.java.simpleName}") + } + } + + suspend fun request(method: String, paramsJson: String?, timeoutMs: Long = 15_000): String { + val conn = currentConnection ?: throw IllegalStateException("not connected") + val params = + if (paramsJson.isNullOrBlank()) { + null + } else { + json.parseToJsonElement(paramsJson) + } + val res = conn.request(method, params, timeoutMs) + if (res.ok) return res.payloadJson ?: "" + val err = res.error + throw IllegalStateException("${err?.code ?: "UNAVAILABLE"}: ${err?.message ?: "request failed"}") + } + + private data class RpcResponse(val id: String, val ok: Boolean, val payloadJson: String?, val error: ErrorShape?) + + private inner class Connection( + private val endpoint: GatewayEndpoint, + private val token: String?, + private val password: String?, + private val options: GatewayConnectOptions, + private val tls: GatewayTlsParams?, + ) { + private val connectDeferred = CompletableDeferred() + private val closedDeferred = CompletableDeferred() + private val isClosed = AtomicBoolean(false) + private val connectNonceDeferred = CompletableDeferred() + private val client: OkHttpClient = buildClient() + private var socket: WebSocket? = null + private val loggerTag = "OpenClawGateway" + + val remoteAddress: String = + if (endpoint.host.contains(":")) { + "[${endpoint.host}]:${endpoint.port}" + } else { + "${endpoint.host}:${endpoint.port}" + } + + suspend fun connect() { + val scheme = if (tls != null) "wss" else "ws" + val url = "$scheme://${endpoint.host}:${endpoint.port}" + val request = Request.Builder().url(url).build() + socket = client.newWebSocket(request, Listener()) + try { + connectDeferred.await() + } catch (err: Throwable) { + throw err + } + } + + suspend fun request(method: String, params: JsonElement?, timeoutMs: Long): RpcResponse { + val id = UUID.randomUUID().toString() + val deferred = CompletableDeferred() + pending[id] = deferred + val frame = + buildJsonObject { + put("type", JsonPrimitive("req")) + put("id", JsonPrimitive(id)) + put("method", JsonPrimitive(method)) + if (params != null) put("params", params) + } + sendJson(frame) + return try { + withTimeout(timeoutMs) { deferred.await() } + } catch (err: TimeoutCancellationException) { + pending.remove(id) + throw IllegalStateException("request timeout") + } + } + + suspend fun sendJson(obj: JsonObject) { + val jsonString = obj.toString() + writeLock.withLock { + socket?.send(jsonString) + } + } + + suspend fun awaitClose() = closedDeferred.await() + + fun closeQuietly() { + if (isClosed.compareAndSet(false, true)) { + socket?.close(1000, "bye") + socket = null + closedDeferred.complete(Unit) + } + } + + private fun buildClient(): OkHttpClient { + val builder = OkHttpClient.Builder() + val tlsConfig = buildGatewayTlsConfig(tls) { fingerprint -> + onTlsFingerprint?.invoke(tls?.stableId ?: endpoint.stableId, fingerprint) + } + if (tlsConfig != null) { + builder.sslSocketFactory(tlsConfig.sslSocketFactory, tlsConfig.trustManager) + builder.hostnameVerifier(tlsConfig.hostnameVerifier) + } + return builder.build() + } + + private inner class Listener : WebSocketListener() { + override fun onOpen(webSocket: WebSocket, response: Response) { + scope.launch { + try { + val nonce = awaitConnectNonce() + sendConnect(nonce) + } catch (err: Throwable) { + connectDeferred.completeExceptionally(err) + closeQuietly() + } + } + } + + override fun onMessage(webSocket: WebSocket, text: String) { + scope.launch { handleMessage(text) } + } + + override fun onFailure(webSocket: WebSocket, t: Throwable, response: Response?) { + if (!connectDeferred.isCompleted) { + connectDeferred.completeExceptionally(t) + } + if (isClosed.compareAndSet(false, true)) { + failPending() + closedDeferred.complete(Unit) + onDisconnected("Gateway error: ${t.message ?: t::class.java.simpleName}") + } + } + + override fun onClosed(webSocket: WebSocket, code: Int, reason: String) { + if (!connectDeferred.isCompleted) { + connectDeferred.completeExceptionally(IllegalStateException("Gateway closed: $reason")) + } + if (isClosed.compareAndSet(false, true)) { + failPending() + closedDeferred.complete(Unit) + onDisconnected("Gateway closed: $reason") + } + } + } + + private suspend fun sendConnect(connectNonce: String?) { + val identity = identityStore.loadOrCreate() + val storedToken = deviceAuthStore.loadToken(identity.deviceId, options.role) + val trimmedToken = token?.trim().orEmpty() + val authToken = if (storedToken.isNullOrBlank()) trimmedToken else storedToken + val canFallbackToShared = !storedToken.isNullOrBlank() && trimmedToken.isNotBlank() + val payload = buildConnectParams(identity, connectNonce, authToken, password?.trim()) + val res = request("connect", payload, timeoutMs = 8_000) + if (!res.ok) { + val msg = res.error?.message ?: "connect failed" + if (canFallbackToShared) { + deviceAuthStore.clearToken(identity.deviceId, options.role) + } + throw IllegalStateException(msg) + } + val payloadJson = res.payloadJson ?: throw IllegalStateException("connect failed: missing payload") + val obj = json.parseToJsonElement(payloadJson).asObjectOrNull() ?: throw IllegalStateException("connect failed") + val serverName = obj["server"].asObjectOrNull()?.get("host").asStringOrNull() + val authObj = obj["auth"].asObjectOrNull() + val deviceToken = authObj?.get("deviceToken").asStringOrNull() + val authRole = authObj?.get("role").asStringOrNull() ?: options.role + if (!deviceToken.isNullOrBlank()) { + deviceAuthStore.saveToken(identity.deviceId, authRole, deviceToken) + } + val rawCanvas = obj["canvasHostUrl"].asStringOrNull() + canvasHostUrl = normalizeCanvasHostUrl(rawCanvas, endpoint) + val sessionDefaults = + obj["snapshot"].asObjectOrNull() + ?.get("sessionDefaults").asObjectOrNull() + mainSessionKey = sessionDefaults?.get("mainSessionKey").asStringOrNull() + onConnected(serverName, remoteAddress, mainSessionKey) + connectDeferred.complete(Unit) + } + + private fun buildConnectParams( + identity: DeviceIdentity, + connectNonce: String?, + authToken: String, + authPassword: String?, + ): JsonObject { + val client = options.client + val locale = Locale.getDefault().toLanguageTag() + val clientObj = + buildJsonObject { + put("id", JsonPrimitive(client.id)) + client.displayName?.let { put("displayName", JsonPrimitive(it)) } + put("version", JsonPrimitive(client.version)) + put("platform", JsonPrimitive(client.platform)) + put("mode", JsonPrimitive(client.mode)) + client.instanceId?.let { put("instanceId", JsonPrimitive(it)) } + client.deviceFamily?.let { put("deviceFamily", JsonPrimitive(it)) } + client.modelIdentifier?.let { put("modelIdentifier", JsonPrimitive(it)) } + } + + val password = authPassword?.trim().orEmpty() + val authJson = + when { + authToken.isNotEmpty() -> + buildJsonObject { + put("token", JsonPrimitive(authToken)) + } + password.isNotEmpty() -> + buildJsonObject { + put("password", JsonPrimitive(password)) + } + else -> null + } + + val signedAtMs = System.currentTimeMillis() + val payload = + buildDeviceAuthPayload( + deviceId = identity.deviceId, + clientId = client.id, + clientMode = client.mode, + role = options.role, + scopes = options.scopes, + signedAtMs = signedAtMs, + token = if (authToken.isNotEmpty()) authToken else null, + nonce = connectNonce, + ) + val signature = identityStore.signPayload(payload, identity) + val publicKey = identityStore.publicKeyBase64Url(identity) + val deviceJson = + if (!signature.isNullOrBlank() && !publicKey.isNullOrBlank()) { + buildJsonObject { + put("id", JsonPrimitive(identity.deviceId)) + put("publicKey", JsonPrimitive(publicKey)) + put("signature", JsonPrimitive(signature)) + put("signedAt", JsonPrimitive(signedAtMs)) + if (!connectNonce.isNullOrBlank()) { + put("nonce", JsonPrimitive(connectNonce)) + } + } + } else { + null + } + + return buildJsonObject { + put("minProtocol", JsonPrimitive(GATEWAY_PROTOCOL_VERSION)) + put("maxProtocol", JsonPrimitive(GATEWAY_PROTOCOL_VERSION)) + put("client", clientObj) + if (options.caps.isNotEmpty()) put("caps", JsonArray(options.caps.map(::JsonPrimitive))) + if (options.commands.isNotEmpty()) put("commands", JsonArray(options.commands.map(::JsonPrimitive))) + if (options.permissions.isNotEmpty()) { + put( + "permissions", + buildJsonObject { + options.permissions.forEach { (key, value) -> + put(key, JsonPrimitive(value)) + } + }, + ) + } + put("role", JsonPrimitive(options.role)) + if (options.scopes.isNotEmpty()) put("scopes", JsonArray(options.scopes.map(::JsonPrimitive))) + authJson?.let { put("auth", it) } + deviceJson?.let { put("device", it) } + put("locale", JsonPrimitive(locale)) + options.userAgent?.trim()?.takeIf { it.isNotEmpty() }?.let { + put("userAgent", JsonPrimitive(it)) + } + } + } + + private suspend fun handleMessage(text: String) { + val frame = json.parseToJsonElement(text).asObjectOrNull() ?: return + when (frame["type"].asStringOrNull()) { + "res" -> handleResponse(frame) + "event" -> handleEvent(frame) + } + } + + private fun handleResponse(frame: JsonObject) { + val id = frame["id"].asStringOrNull() ?: return + val ok = frame["ok"].asBooleanOrNull() ?: false + val payloadJson = frame["payload"]?.let { payload -> payload.toString() } + val error = + frame["error"]?.asObjectOrNull()?.let { obj -> + val code = obj["code"].asStringOrNull() ?: "UNAVAILABLE" + val msg = obj["message"].asStringOrNull() ?: "request failed" + ErrorShape(code, msg) + } + pending.remove(id)?.complete(RpcResponse(id, ok, payloadJson, error)) + } + + private fun handleEvent(frame: JsonObject) { + val event = frame["event"].asStringOrNull() ?: return + val payloadJson = + frame["payload"]?.let { it.toString() } ?: frame["payloadJSON"].asStringOrNull() + if (event == "connect.challenge") { + val nonce = extractConnectNonce(payloadJson) + if (!connectNonceDeferred.isCompleted) { + connectNonceDeferred.complete(nonce) + } + return + } + if (event == "node.invoke.request" && payloadJson != null && onInvoke != null) { + handleInvokeEvent(payloadJson) + return + } + onEvent(event, payloadJson) + } + + private suspend fun awaitConnectNonce(): String? { + if (isLoopbackHost(endpoint.host)) return null + return try { + withTimeout(2_000) { connectNonceDeferred.await() } + } catch (_: Throwable) { + null + } + } + + private fun extractConnectNonce(payloadJson: String?): String? { + if (payloadJson.isNullOrBlank()) return null + val obj = parseJsonOrNull(payloadJson)?.asObjectOrNull() ?: return null + return obj["nonce"].asStringOrNull() + } + + private fun handleInvokeEvent(payloadJson: String) { + val payload = + try { + json.parseToJsonElement(payloadJson).asObjectOrNull() + } catch (_: Throwable) { + null + } ?: return + val id = payload["id"].asStringOrNull() ?: return + val nodeId = payload["nodeId"].asStringOrNull() ?: return + val command = payload["command"].asStringOrNull() ?: return + val params = + payload["paramsJSON"].asStringOrNull() + ?: payload["params"]?.let { value -> if (value is JsonNull) null else value.toString() } + val timeoutMs = payload["timeoutMs"].asLongOrNull() + scope.launch { + val result = + try { + onInvoke?.invoke(InvokeRequest(id, nodeId, command, params, timeoutMs)) + ?: InvokeResult.error("UNAVAILABLE", "invoke handler missing") + } catch (err: Throwable) { + invokeErrorFromThrowable(err) + } + sendInvokeResult(id, nodeId, result) + } + } + + private suspend fun sendInvokeResult(id: String, nodeId: String, result: InvokeResult) { + val parsedPayload = result.payloadJson?.let { parseJsonOrNull(it) } + val params = + buildJsonObject { + put("id", JsonPrimitive(id)) + put("nodeId", JsonPrimitive(nodeId)) + put("ok", JsonPrimitive(result.ok)) + if (parsedPayload != null) { + put("payload", parsedPayload) + } else if (result.payloadJson != null) { + put("payloadJSON", JsonPrimitive(result.payloadJson)) + } + result.error?.let { err -> + put( + "error", + buildJsonObject { + put("code", JsonPrimitive(err.code)) + put("message", JsonPrimitive(err.message)) + }, + ) + } + } + try { + request("node.invoke.result", params, timeoutMs = 15_000) + } catch (err: Throwable) { + Log.w(loggerTag, "node.invoke.result failed: ${err.message ?: err::class.java.simpleName}") + } + } + + private fun invokeErrorFromThrowable(err: Throwable): InvokeResult { + val msg = err.message?.trim().takeIf { !it.isNullOrEmpty() } ?: err::class.java.simpleName + val parts = msg.split(":", limit = 2) + if (parts.size == 2) { + val code = parts[0].trim() + val rest = parts[1].trim() + if (code.isNotEmpty() && code.all { it.isUpperCase() || it == '_' }) { + return InvokeResult.error(code = code, message = rest.ifEmpty { msg }) + } + } + return InvokeResult.error(code = "UNAVAILABLE", message = msg) + } + + private fun failPending() { + for ((_, waiter) in pending) { + waiter.cancel() + } + pending.clear() + } + } + + private suspend fun runLoop() { + var attempt = 0 + while (scope.isActive) { + val target = desired + if (target == null) { + currentConnection?.closeQuietly() + currentConnection = null + delay(250) + continue + } + + try { + onDisconnected(if (attempt == 0) "Connecting…" else "Reconnecting…") + connectOnce(target) + attempt = 0 + } catch (err: Throwable) { + attempt += 1 + onDisconnected("Gateway error: ${err.message ?: err::class.java.simpleName}") + val sleepMs = minOf(8_000L, (350.0 * Math.pow(1.7, attempt.toDouble())).toLong()) + delay(sleepMs) + } + } + } + + private suspend fun connectOnce(target: DesiredConnection) = withContext(Dispatchers.IO) { + val conn = Connection(target.endpoint, target.token, target.password, target.options, target.tls) + currentConnection = conn + try { + conn.connect() + conn.awaitClose() + } finally { + currentConnection = null + canvasHostUrl = null + mainSessionKey = null + } + } + + private fun buildDeviceAuthPayload( + deviceId: String, + clientId: String, + clientMode: String, + role: String, + scopes: List, + signedAtMs: Long, + token: String?, + nonce: String?, + ): String { + val scopeString = scopes.joinToString(",") + val authToken = token.orEmpty() + val version = if (nonce.isNullOrBlank()) "v1" else "v2" + val parts = + mutableListOf( + version, + deviceId, + clientId, + clientMode, + role, + scopeString, + signedAtMs.toString(), + authToken, + ) + if (!nonce.isNullOrBlank()) { + parts.add(nonce) + } + return parts.joinToString("|") + } + + private fun normalizeCanvasHostUrl(raw: String?, endpoint: GatewayEndpoint): String? { + val trimmed = raw?.trim().orEmpty() + val parsed = trimmed.takeIf { it.isNotBlank() }?.let { runCatching { java.net.URI(it) }.getOrNull() } + val host = parsed?.host?.trim().orEmpty() + val port = parsed?.port ?: -1 + val scheme = parsed?.scheme?.trim().orEmpty().ifBlank { "http" } + + if (trimmed.isNotBlank() && !isLoopbackHost(host)) { + return trimmed + } + + val fallbackHost = + endpoint.tailnetDns?.trim().takeIf { !it.isNullOrEmpty() } + ?: endpoint.lanHost?.trim().takeIf { !it.isNullOrEmpty() } + ?: endpoint.host.trim() + if (fallbackHost.isEmpty()) return trimmed.ifBlank { null } + + val fallbackPort = endpoint.canvasPort ?: if (port > 0) port else 18793 + val formattedHost = if (fallbackHost.contains(":")) "[${fallbackHost}]" else fallbackHost + return "$scheme://$formattedHost:$fallbackPort" + } + + private fun isLoopbackHost(raw: String?): Boolean { + val host = raw?.trim()?.lowercase().orEmpty() + if (host.isEmpty()) return false + if (host == "localhost") return true + if (host == "::1") return true + if (host == "0.0.0.0" || host == "::") return true + return host.startsWith("127.") + } +} + +private fun JsonElement?.asObjectOrNull(): JsonObject? = this as? JsonObject + +private fun JsonElement?.asStringOrNull(): String? = + when (this) { + is JsonNull -> null + is JsonPrimitive -> content + else -> null + } + +private fun JsonElement?.asBooleanOrNull(): Boolean? = + when (this) { + is JsonPrimitive -> { + val c = content.trim() + when { + c.equals("true", ignoreCase = true) -> true + c.equals("false", ignoreCase = true) -> false + else -> null + } + } + else -> null + } + +private fun JsonElement?.asLongOrNull(): Long? = + when (this) { + is JsonPrimitive -> content.toLongOrNull() + else -> null + } + +private fun parseJsonOrNull(payload: String): JsonElement? { + val trimmed = payload.trim() + if (trimmed.isEmpty()) return null + return try { + Json.parseToJsonElement(trimmed) + } catch (_: Throwable) { + null + } +} diff --git a/apps/android/app/src/main/java/ai/openclaw/android/gateway/GatewayTls.kt b/apps/android/app/src/main/java/ai/openclaw/android/gateway/GatewayTls.kt new file mode 100644 index 0000000000000000000000000000000000000000..dc17aa73292561e7e5e0f0189b7828575eac1f88 --- /dev/null +++ b/apps/android/app/src/main/java/ai/openclaw/android/gateway/GatewayTls.kt @@ -0,0 +1,90 @@ +package ai.openclaw.android.gateway + +import android.annotation.SuppressLint +import java.security.MessageDigest +import java.security.SecureRandom +import java.security.cert.CertificateException +import java.security.cert.X509Certificate +import javax.net.ssl.HostnameVerifier +import javax.net.ssl.SSLContext +import javax.net.ssl.SSLSocketFactory +import javax.net.ssl.TrustManagerFactory +import javax.net.ssl.X509TrustManager + +data class GatewayTlsParams( + val required: Boolean, + val expectedFingerprint: String?, + val allowTOFU: Boolean, + val stableId: String, +) + +data class GatewayTlsConfig( + val sslSocketFactory: SSLSocketFactory, + val trustManager: X509TrustManager, + val hostnameVerifier: HostnameVerifier, +) + +fun buildGatewayTlsConfig( + params: GatewayTlsParams?, + onStore: ((String) -> Unit)? = null, +): GatewayTlsConfig? { + if (params == null) return null + val expected = params.expectedFingerprint?.let(::normalizeFingerprint) + val defaultTrust = defaultTrustManager() + @SuppressLint("CustomX509TrustManager") + val trustManager = + object : X509TrustManager { + override fun checkClientTrusted(chain: Array, authType: String) { + defaultTrust.checkClientTrusted(chain, authType) + } + + override fun checkServerTrusted(chain: Array, authType: String) { + if (chain.isEmpty()) throw CertificateException("empty certificate chain") + val fingerprint = sha256Hex(chain[0].encoded) + if (expected != null) { + if (fingerprint != expected) { + throw CertificateException("gateway TLS fingerprint mismatch") + } + return + } + if (params.allowTOFU) { + onStore?.invoke(fingerprint) + return + } + defaultTrust.checkServerTrusted(chain, authType) + } + + override fun getAcceptedIssuers(): Array = defaultTrust.acceptedIssuers + } + + val context = SSLContext.getInstance("TLS") + context.init(null, arrayOf(trustManager), SecureRandom()) + return GatewayTlsConfig( + sslSocketFactory = context.socketFactory, + trustManager = trustManager, + hostnameVerifier = HostnameVerifier { _, _ -> true }, + ) +} + +private fun defaultTrustManager(): X509TrustManager { + val factory = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm()) + factory.init(null as java.security.KeyStore?) + val trust = + factory.trustManagers.firstOrNull { it is X509TrustManager } as? X509TrustManager + return trust ?: throw IllegalStateException("No default X509TrustManager found") +} + +private fun sha256Hex(data: ByteArray): String { + val digest = MessageDigest.getInstance("SHA-256").digest(data) + val out = StringBuilder(digest.size * 2) + for (byte in digest) { + out.append(String.format("%02x", byte)) + } + return out.toString() +} + +private fun normalizeFingerprint(raw: String): String { + val stripped = raw.trim() + .replace(Regex("^sha-?256\\s*:?\\s*", RegexOption.IGNORE_CASE), "") + return stripped.lowercase().filter { it in '0'..'9' || it in 'a'..'f' } +} diff --git a/apps/android/app/src/main/java/ai/openclaw/android/node/CameraCaptureManager.kt b/apps/android/app/src/main/java/ai/openclaw/android/node/CameraCaptureManager.kt new file mode 100644 index 0000000000000000000000000000000000000000..536c8cbda88fdc79b8f61ada3863127cbd59c128 --- /dev/null +++ b/apps/android/app/src/main/java/ai/openclaw/android/node/CameraCaptureManager.kt @@ -0,0 +1,316 @@ +package ai.openclaw.android.node + +import android.Manifest +import android.content.Context +import android.annotation.SuppressLint +import android.graphics.Bitmap +import android.graphics.BitmapFactory +import android.graphics.Matrix +import android.util.Base64 +import android.content.pm.PackageManager +import androidx.exifinterface.media.ExifInterface +import androidx.lifecycle.LifecycleOwner +import androidx.camera.core.CameraSelector +import androidx.camera.core.ImageCapture +import androidx.camera.core.ImageCaptureException +import androidx.camera.lifecycle.ProcessCameraProvider +import androidx.camera.video.FileOutputOptions +import androidx.camera.video.Recorder +import androidx.camera.video.Recording +import androidx.camera.video.VideoCapture +import androidx.camera.video.VideoRecordEvent +import androidx.core.content.ContextCompat +import androidx.core.content.ContextCompat.checkSelfPermission +import androidx.core.graphics.scale +import ai.openclaw.android.PermissionRequester +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.suspendCancellableCoroutine +import kotlinx.coroutines.withTimeout +import kotlinx.coroutines.withContext +import java.io.ByteArrayOutputStream +import java.io.File +import java.util.concurrent.Executor +import kotlin.math.roundToInt +import kotlin.coroutines.resume +import kotlin.coroutines.resumeWithException + +class CameraCaptureManager(private val context: Context) { + data class Payload(val payloadJson: String) + + @Volatile private var lifecycleOwner: LifecycleOwner? = null + @Volatile private var permissionRequester: PermissionRequester? = null + + fun attachLifecycleOwner(owner: LifecycleOwner) { + lifecycleOwner = owner + } + + fun attachPermissionRequester(requester: PermissionRequester) { + permissionRequester = requester + } + + private suspend fun ensureCameraPermission() { + val granted = checkSelfPermission(context, Manifest.permission.CAMERA) == PackageManager.PERMISSION_GRANTED + if (granted) return + + val requester = permissionRequester + ?: throw IllegalStateException("CAMERA_PERMISSION_REQUIRED: grant Camera permission") + val results = requester.requestIfMissing(listOf(Manifest.permission.CAMERA)) + if (results[Manifest.permission.CAMERA] != true) { + throw IllegalStateException("CAMERA_PERMISSION_REQUIRED: grant Camera permission") + } + } + + private suspend fun ensureMicPermission() { + val granted = checkSelfPermission(context, Manifest.permission.RECORD_AUDIO) == PackageManager.PERMISSION_GRANTED + if (granted) return + + val requester = permissionRequester + ?: throw IllegalStateException("MIC_PERMISSION_REQUIRED: grant Microphone permission") + val results = requester.requestIfMissing(listOf(Manifest.permission.RECORD_AUDIO)) + if (results[Manifest.permission.RECORD_AUDIO] != true) { + throw IllegalStateException("MIC_PERMISSION_REQUIRED: grant Microphone permission") + } + } + + suspend fun snap(paramsJson: String?): Payload = + withContext(Dispatchers.Main) { + ensureCameraPermission() + val owner = lifecycleOwner ?: throw IllegalStateException("UNAVAILABLE: camera not ready") + val facing = parseFacing(paramsJson) ?: "front" + val quality = (parseQuality(paramsJson) ?: 0.9).coerceIn(0.1, 1.0) + val maxWidth = parseMaxWidth(paramsJson) + + val provider = context.cameraProvider() + val capture = ImageCapture.Builder().build() + val selector = + if (facing == "front") CameraSelector.DEFAULT_FRONT_CAMERA else CameraSelector.DEFAULT_BACK_CAMERA + + provider.unbindAll() + provider.bindToLifecycle(owner, selector, capture) + + val (bytes, orientation) = capture.takeJpegWithExif(context.mainExecutor()) + val decoded = BitmapFactory.decodeByteArray(bytes, 0, bytes.size) + ?: throw IllegalStateException("UNAVAILABLE: failed to decode captured image") + val rotated = rotateBitmapByExif(decoded, orientation) + val scaled = + if (maxWidth != null && maxWidth > 0 && rotated.width > maxWidth) { + val h = + (rotated.height.toDouble() * (maxWidth.toDouble() / rotated.width.toDouble())) + .toInt() + .coerceAtLeast(1) + rotated.scale(maxWidth, h) + } else { + rotated + } + + val maxPayloadBytes = 5 * 1024 * 1024 + // Base64 inflates payloads by ~4/3; cap encoded bytes so the payload stays under 5MB (API limit). + val maxEncodedBytes = (maxPayloadBytes / 4) * 3 + val result = + JpegSizeLimiter.compressToLimit( + initialWidth = scaled.width, + initialHeight = scaled.height, + startQuality = (quality * 100.0).roundToInt().coerceIn(10, 100), + maxBytes = maxEncodedBytes, + encode = { width, height, q -> + val bitmap = + if (width == scaled.width && height == scaled.height) { + scaled + } else { + scaled.scale(width, height) + } + val out = ByteArrayOutputStream() + if (!bitmap.compress(Bitmap.CompressFormat.JPEG, q, out)) { + if (bitmap !== scaled) bitmap.recycle() + throw IllegalStateException("UNAVAILABLE: failed to encode JPEG") + } + if (bitmap !== scaled) { + bitmap.recycle() + } + out.toByteArray() + }, + ) + val base64 = Base64.encodeToString(result.bytes, Base64.NO_WRAP) + Payload( + """{"format":"jpg","base64":"$base64","width":${result.width},"height":${result.height}}""", + ) + } + + @SuppressLint("MissingPermission") + suspend fun clip(paramsJson: String?): Payload = + withContext(Dispatchers.Main) { + ensureCameraPermission() + val owner = lifecycleOwner ?: throw IllegalStateException("UNAVAILABLE: camera not ready") + val facing = parseFacing(paramsJson) ?: "front" + val durationMs = (parseDurationMs(paramsJson) ?: 3_000).coerceIn(200, 60_000) + val includeAudio = parseIncludeAudio(paramsJson) ?: true + if (includeAudio) ensureMicPermission() + + val provider = context.cameraProvider() + val recorder = Recorder.Builder().build() + val videoCapture = VideoCapture.withOutput(recorder) + val selector = + if (facing == "front") CameraSelector.DEFAULT_FRONT_CAMERA else CameraSelector.DEFAULT_BACK_CAMERA + + provider.unbindAll() + provider.bindToLifecycle(owner, selector, videoCapture) + + val file = File.createTempFile("openclaw-clip-", ".mp4") + val outputOptions = FileOutputOptions.Builder(file).build() + + val finalized = kotlinx.coroutines.CompletableDeferred() + val recording: Recording = + videoCapture.output + .prepareRecording(context, outputOptions) + .apply { + if (includeAudio) withAudioEnabled() + } + .start(context.mainExecutor()) { event -> + if (event is VideoRecordEvent.Finalize) { + finalized.complete(event) + } + } + + try { + kotlinx.coroutines.delay(durationMs.toLong()) + } finally { + recording.stop() + } + + val finalizeEvent = + try { + withTimeout(10_000) { finalized.await() } + } catch (err: Throwable) { + file.delete() + throw IllegalStateException("UNAVAILABLE: camera clip finalize timed out") + } + if (finalizeEvent.hasError()) { + file.delete() + throw IllegalStateException("UNAVAILABLE: camera clip failed") + } + + val bytes = file.readBytes() + file.delete() + val base64 = Base64.encodeToString(bytes, Base64.NO_WRAP) + Payload( + """{"format":"mp4","base64":"$base64","durationMs":$durationMs,"hasAudio":${includeAudio}}""", + ) + } + + private fun rotateBitmapByExif(bitmap: Bitmap, orientation: Int): Bitmap { + val matrix = Matrix() + when (orientation) { + ExifInterface.ORIENTATION_ROTATE_90 -> matrix.postRotate(90f) + ExifInterface.ORIENTATION_ROTATE_180 -> matrix.postRotate(180f) + ExifInterface.ORIENTATION_ROTATE_270 -> matrix.postRotate(270f) + ExifInterface.ORIENTATION_FLIP_HORIZONTAL -> matrix.postScale(-1f, 1f) + ExifInterface.ORIENTATION_FLIP_VERTICAL -> matrix.postScale(1f, -1f) + ExifInterface.ORIENTATION_TRANSPOSE -> { + matrix.postRotate(90f) + matrix.postScale(-1f, 1f) + } + ExifInterface.ORIENTATION_TRANSVERSE -> { + matrix.postRotate(-90f) + matrix.postScale(-1f, 1f) + } + else -> return bitmap + } + val rotated = Bitmap.createBitmap(bitmap, 0, 0, bitmap.width, bitmap.height, matrix, true) + if (rotated !== bitmap) { + bitmap.recycle() + } + return rotated + } + + private fun parseFacing(paramsJson: String?): String? = + when { + paramsJson?.contains("\"front\"") == true -> "front" + paramsJson?.contains("\"back\"") == true -> "back" + else -> null + } + + private fun parseQuality(paramsJson: String?): Double? = + parseNumber(paramsJson, key = "quality")?.toDoubleOrNull() + + private fun parseMaxWidth(paramsJson: String?): Int? = + parseNumber(paramsJson, key = "maxWidth")?.toIntOrNull() + + private fun parseDurationMs(paramsJson: String?): Int? = + parseNumber(paramsJson, key = "durationMs")?.toIntOrNull() + + private fun parseIncludeAudio(paramsJson: String?): Boolean? { + val raw = paramsJson ?: return null + val key = "\"includeAudio\"" + val idx = raw.indexOf(key) + if (idx < 0) return null + val colon = raw.indexOf(':', idx + key.length) + if (colon < 0) return null + val tail = raw.substring(colon + 1).trimStart() + return when { + tail.startsWith("true") -> true + tail.startsWith("false") -> false + else -> null + } + } + + private fun parseNumber(paramsJson: String?, key: String): String? { + val raw = paramsJson ?: return null + val needle = "\"$key\"" + val idx = raw.indexOf(needle) + if (idx < 0) return null + val colon = raw.indexOf(':', idx + needle.length) + if (colon < 0) return null + val tail = raw.substring(colon + 1).trimStart() + return tail.takeWhile { it.isDigit() || it == '.' } + } + + private fun Context.mainExecutor(): Executor = ContextCompat.getMainExecutor(this) +} + +private suspend fun Context.cameraProvider(): ProcessCameraProvider = + suspendCancellableCoroutine { cont -> + val future = ProcessCameraProvider.getInstance(this) + future.addListener( + { + try { + cont.resume(future.get()) + } catch (e: Exception) { + cont.resumeWithException(e) + } + }, + ContextCompat.getMainExecutor(this), + ) + } + +/** Returns (jpegBytes, exifOrientation) so caller can rotate the decoded bitmap. */ +private suspend fun ImageCapture.takeJpegWithExif(executor: Executor): Pair = + suspendCancellableCoroutine { cont -> + val file = File.createTempFile("openclaw-snap-", ".jpg") + val options = ImageCapture.OutputFileOptions.Builder(file).build() + takePicture( + options, + executor, + object : ImageCapture.OnImageSavedCallback { + override fun onError(exception: ImageCaptureException) { + file.delete() + cont.resumeWithException(exception) + } + + override fun onImageSaved(outputFileResults: ImageCapture.OutputFileResults) { + try { + val exif = ExifInterface(file.absolutePath) + val orientation = exif.getAttributeInt( + ExifInterface.TAG_ORIENTATION, + ExifInterface.ORIENTATION_NORMAL, + ) + val bytes = file.readBytes() + cont.resume(Pair(bytes, orientation)) + } catch (e: Exception) { + cont.resumeWithException(e) + } finally { + file.delete() + } + } + }, + ) + } diff --git a/apps/android/app/src/main/java/ai/openclaw/android/node/CanvasController.kt b/apps/android/app/src/main/java/ai/openclaw/android/node/CanvasController.kt new file mode 100644 index 0000000000000000000000000000000000000000..c46770a6367f1d849b8dce403a4ebb625147e04d --- /dev/null +++ b/apps/android/app/src/main/java/ai/openclaw/android/node/CanvasController.kt @@ -0,0 +1,264 @@ +package ai.openclaw.android.node + +import android.graphics.Bitmap +import android.graphics.Canvas +import android.os.Looper +import android.util.Log +import android.webkit.WebView +import androidx.core.graphics.createBitmap +import androidx.core.graphics.scale +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.suspendCancellableCoroutine +import kotlinx.coroutines.withContext +import java.io.ByteArrayOutputStream +import android.util.Base64 +import org.json.JSONObject +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.JsonElement +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.JsonPrimitive +import ai.openclaw.android.BuildConfig +import kotlin.coroutines.resume + +class CanvasController { + enum class SnapshotFormat(val rawValue: String) { + Png("png"), + Jpeg("jpeg"), + } + + @Volatile private var webView: WebView? = null + @Volatile private var url: String? = null + @Volatile private var debugStatusEnabled: Boolean = false + @Volatile private var debugStatusTitle: String? = null + @Volatile private var debugStatusSubtitle: String? = null + + private val scaffoldAssetUrl = "file:///android_asset/CanvasScaffold/scaffold.html" + + private fun clampJpegQuality(quality: Double?): Int { + val q = (quality ?: 0.82).coerceIn(0.1, 1.0) + return (q * 100.0).toInt().coerceIn(1, 100) + } + + fun attach(webView: WebView) { + this.webView = webView + reload() + applyDebugStatus() + } + + fun navigate(url: String) { + val trimmed = url.trim() + this.url = if (trimmed.isBlank() || trimmed == "/") null else trimmed + reload() + } + + fun currentUrl(): String? = url + + fun isDefaultCanvas(): Boolean = url == null + + fun setDebugStatusEnabled(enabled: Boolean) { + debugStatusEnabled = enabled + applyDebugStatus() + } + + fun setDebugStatus(title: String?, subtitle: String?) { + debugStatusTitle = title + debugStatusSubtitle = subtitle + applyDebugStatus() + } + + fun onPageFinished() { + applyDebugStatus() + } + + private inline fun withWebViewOnMain(crossinline block: (WebView) -> Unit) { + val wv = webView ?: return + if (Looper.myLooper() == Looper.getMainLooper()) { + block(wv) + } else { + wv.post { block(wv) } + } + } + + private fun reload() { + val currentUrl = url + withWebViewOnMain { wv -> + if (currentUrl == null) { + if (BuildConfig.DEBUG) { + Log.d("OpenClawCanvas", "load scaffold: $scaffoldAssetUrl") + } + wv.loadUrl(scaffoldAssetUrl) + } else { + if (BuildConfig.DEBUG) { + Log.d("OpenClawCanvas", "load url: $currentUrl") + } + wv.loadUrl(currentUrl) + } + } + } + + private fun applyDebugStatus() { + val enabled = debugStatusEnabled + val title = debugStatusTitle + val subtitle = debugStatusSubtitle + withWebViewOnMain { wv -> + val titleJs = title?.let { JSONObject.quote(it) } ?: "null" + val subtitleJs = subtitle?.let { JSONObject.quote(it) } ?: "null" + val js = """ + (() => { + try { + const api = globalThis.__openclaw; + if (!api) return; + if (typeof api.setDebugStatusEnabled === 'function') { + api.setDebugStatusEnabled(${if (enabled) "true" else "false"}); + } + if (!${if (enabled) "true" else "false"}) return; + if (typeof api.setStatus === 'function') { + api.setStatus($titleJs, $subtitleJs); + } + } catch (_) {} + })(); + """.trimIndent() + wv.evaluateJavascript(js, null) + } + } + + suspend fun eval(javaScript: String): String = + withContext(Dispatchers.Main) { + val wv = webView ?: throw IllegalStateException("no webview") + suspendCancellableCoroutine { cont -> + wv.evaluateJavascript(javaScript) { result -> + cont.resume(result ?: "") + } + } + } + + suspend fun snapshotPngBase64(maxWidth: Int?): String = + withContext(Dispatchers.Main) { + val wv = webView ?: throw IllegalStateException("no webview") + val bmp = wv.captureBitmap() + val scaled = + if (maxWidth != null && maxWidth > 0 && bmp.width > maxWidth) { + val h = (bmp.height.toDouble() * (maxWidth.toDouble() / bmp.width.toDouble())).toInt().coerceAtLeast(1) + bmp.scale(maxWidth, h) + } else { + bmp + } + + val out = ByteArrayOutputStream() + scaled.compress(Bitmap.CompressFormat.PNG, 100, out) + Base64.encodeToString(out.toByteArray(), Base64.NO_WRAP) + } + + suspend fun snapshotBase64(format: SnapshotFormat, quality: Double?, maxWidth: Int?): String = + withContext(Dispatchers.Main) { + val wv = webView ?: throw IllegalStateException("no webview") + val bmp = wv.captureBitmap() + val scaled = + if (maxWidth != null && maxWidth > 0 && bmp.width > maxWidth) { + val h = (bmp.height.toDouble() * (maxWidth.toDouble() / bmp.width.toDouble())).toInt().coerceAtLeast(1) + bmp.scale(maxWidth, h) + } else { + bmp + } + + val out = ByteArrayOutputStream() + val (compressFormat, compressQuality) = + when (format) { + SnapshotFormat.Png -> Bitmap.CompressFormat.PNG to 100 + SnapshotFormat.Jpeg -> Bitmap.CompressFormat.JPEG to clampJpegQuality(quality) + } + scaled.compress(compressFormat, compressQuality, out) + Base64.encodeToString(out.toByteArray(), Base64.NO_WRAP) + } + + private suspend fun WebView.captureBitmap(): Bitmap = + suspendCancellableCoroutine { cont -> + val width = width.coerceAtLeast(1) + val height = height.coerceAtLeast(1) + val bitmap = createBitmap(width, height, Bitmap.Config.ARGB_8888) + + // WebView isn't supported by PixelCopy.request(...) directly; draw() is the most reliable + // cross-version snapshot for this lightweight "canvas" use-case. + draw(Canvas(bitmap)) + cont.resume(bitmap) + } + + companion object { + data class SnapshotParams(val format: SnapshotFormat, val quality: Double?, val maxWidth: Int?) + + fun parseNavigateUrl(paramsJson: String?): String { + val obj = parseParamsObject(paramsJson) ?: return "" + return obj.string("url").trim() + } + + fun parseEvalJs(paramsJson: String?): String? { + val obj = parseParamsObject(paramsJson) ?: return null + val js = obj.string("javaScript").trim() + return js.takeIf { it.isNotBlank() } + } + + fun parseSnapshotMaxWidth(paramsJson: String?): Int? { + val obj = parseParamsObject(paramsJson) ?: return null + if (!obj.containsKey("maxWidth")) return null + val width = obj.int("maxWidth") ?: 0 + return width.takeIf { it > 0 } + } + + fun parseSnapshotFormat(paramsJson: String?): SnapshotFormat { + val obj = parseParamsObject(paramsJson) ?: return SnapshotFormat.Jpeg + val raw = obj.string("format").trim().lowercase() + return when (raw) { + "png" -> SnapshotFormat.Png + "jpeg", "jpg" -> SnapshotFormat.Jpeg + "" -> SnapshotFormat.Jpeg + else -> SnapshotFormat.Jpeg + } + } + + fun parseSnapshotQuality(paramsJson: String?): Double? { + val obj = parseParamsObject(paramsJson) ?: return null + if (!obj.containsKey("quality")) return null + val q = obj.double("quality") ?: Double.NaN + if (!q.isFinite()) return null + return q.coerceIn(0.1, 1.0) + } + + fun parseSnapshotParams(paramsJson: String?): SnapshotParams { + return SnapshotParams( + format = parseSnapshotFormat(paramsJson), + quality = parseSnapshotQuality(paramsJson), + maxWidth = parseSnapshotMaxWidth(paramsJson), + ) + } + + private val json = Json { ignoreUnknownKeys = true } + + private fun parseParamsObject(paramsJson: String?): JsonObject? { + val raw = paramsJson?.trim().orEmpty() + if (raw.isEmpty()) return null + return try { + json.parseToJsonElement(raw).asObjectOrNull() + } catch (_: Throwable) { + null + } + } + + private fun JsonElement?.asObjectOrNull(): JsonObject? = this as? JsonObject + + private fun JsonObject.string(key: String): String { + val prim = this[key] as? JsonPrimitive ?: return "" + val raw = prim.content + return raw.takeIf { it != "null" }.orEmpty() + } + + private fun JsonObject.int(key: String): Int? { + val prim = this[key] as? JsonPrimitive ?: return null + return prim.content.toIntOrNull() + } + + private fun JsonObject.double(key: String): Double? { + val prim = this[key] as? JsonPrimitive ?: return null + return prim.content.toDoubleOrNull() + } + } +} diff --git a/apps/android/app/src/main/java/ai/openclaw/android/node/JpegSizeLimiter.kt b/apps/android/app/src/main/java/ai/openclaw/android/node/JpegSizeLimiter.kt new file mode 100644 index 0000000000000000000000000000000000000000..d6018467e66f2e6041d2534f75c3799acad41a1f --- /dev/null +++ b/apps/android/app/src/main/java/ai/openclaw/android/node/JpegSizeLimiter.kt @@ -0,0 +1,61 @@ +package ai.openclaw.android.node + +import kotlin.math.max +import kotlin.math.min +import kotlin.math.roundToInt + +internal data class JpegSizeLimiterResult( + val bytes: ByteArray, + val width: Int, + val height: Int, + val quality: Int, +) + +internal object JpegSizeLimiter { + fun compressToLimit( + initialWidth: Int, + initialHeight: Int, + startQuality: Int, + maxBytes: Int, + minQuality: Int = 20, + minSize: Int = 256, + scaleStep: Double = 0.85, + maxScaleAttempts: Int = 6, + maxQualityAttempts: Int = 6, + encode: (width: Int, height: Int, quality: Int) -> ByteArray, + ): JpegSizeLimiterResult { + require(initialWidth > 0 && initialHeight > 0) { "Invalid image size" } + require(maxBytes > 0) { "Invalid maxBytes" } + + var width = initialWidth + var height = initialHeight + val clampedStartQuality = startQuality.coerceIn(minQuality, 100) + var best = JpegSizeLimiterResult(bytes = encode(width, height, clampedStartQuality), width = width, height = height, quality = clampedStartQuality) + if (best.bytes.size <= maxBytes) return best + + repeat(maxScaleAttempts) { + var quality = clampedStartQuality + repeat(maxQualityAttempts) { + val bytes = encode(width, height, quality) + best = JpegSizeLimiterResult(bytes = bytes, width = width, height = height, quality = quality) + if (bytes.size <= maxBytes) return best + if (quality <= minQuality) return@repeat + quality = max(minQuality, (quality * 0.75).roundToInt()) + } + + val minScale = (minSize.toDouble() / min(width, height).toDouble()).coerceAtMost(1.0) + val nextScale = max(scaleStep, minScale) + val nextWidth = max(minSize, (width * nextScale).roundToInt()) + val nextHeight = max(minSize, (height * nextScale).roundToInt()) + if (nextWidth == width && nextHeight == height) return@repeat + width = min(nextWidth, width) + height = min(nextHeight, height) + } + + if (best.bytes.size > maxBytes) { + throw IllegalStateException("CAMERA_TOO_LARGE: ${best.bytes.size} bytes > $maxBytes bytes") + } + + return best + } +} diff --git a/apps/android/app/src/main/java/ai/openclaw/android/node/LocationCaptureManager.kt b/apps/android/app/src/main/java/ai/openclaw/android/node/LocationCaptureManager.kt new file mode 100644 index 0000000000000000000000000000000000000000..87762e87fa9acb84803a0784181e59b3b78ce4f0 --- /dev/null +++ b/apps/android/app/src/main/java/ai/openclaw/android/node/LocationCaptureManager.kt @@ -0,0 +1,117 @@ +package ai.openclaw.android.node + +import android.Manifest +import android.content.Context +import android.content.pm.PackageManager +import android.location.Location +import android.location.LocationManager +import android.os.CancellationSignal +import androidx.core.content.ContextCompat +import java.time.Instant +import java.time.format.DateTimeFormatter +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext +import kotlinx.coroutines.withTimeout +import kotlin.coroutines.resume +import kotlin.coroutines.resumeWithException +import kotlinx.coroutines.suspendCancellableCoroutine + +class LocationCaptureManager(private val context: Context) { + data class Payload(val payloadJson: String) + + suspend fun getLocation( + desiredProviders: List, + maxAgeMs: Long?, + timeoutMs: Long, + isPrecise: Boolean, + ): Payload = + withContext(Dispatchers.Main) { + val manager = context.getSystemService(Context.LOCATION_SERVICE) as LocationManager + if (!manager.isProviderEnabled(LocationManager.GPS_PROVIDER) && + !manager.isProviderEnabled(LocationManager.NETWORK_PROVIDER) + ) { + throw IllegalStateException("LOCATION_UNAVAILABLE: no location providers enabled") + } + + val cached = bestLastKnown(manager, desiredProviders, maxAgeMs) + val location = + cached ?: requestCurrent(manager, desiredProviders, timeoutMs) + + val timestamp = DateTimeFormatter.ISO_INSTANT.format(Instant.ofEpochMilli(location.time)) + val source = location.provider + val altitudeMeters = if (location.hasAltitude()) location.altitude else null + val speedMps = if (location.hasSpeed()) location.speed.toDouble() else null + val headingDeg = if (location.hasBearing()) location.bearing.toDouble() else null + Payload( + buildString { + append("{\"lat\":") + append(location.latitude) + append(",\"lon\":") + append(location.longitude) + append(",\"accuracyMeters\":") + append(location.accuracy.toDouble()) + if (altitudeMeters != null) append(",\"altitudeMeters\":").append(altitudeMeters) + if (speedMps != null) append(",\"speedMps\":").append(speedMps) + if (headingDeg != null) append(",\"headingDeg\":").append(headingDeg) + append(",\"timestamp\":\"").append(timestamp).append('"') + append(",\"isPrecise\":").append(isPrecise) + append(",\"source\":\"").append(source).append('"') + append('}') + }, + ) + } + + private fun bestLastKnown( + manager: LocationManager, + providers: List, + maxAgeMs: Long?, + ): Location? { + val fineOk = + ContextCompat.checkSelfPermission(context, Manifest.permission.ACCESS_FINE_LOCATION) == + PackageManager.PERMISSION_GRANTED + val coarseOk = + ContextCompat.checkSelfPermission(context, Manifest.permission.ACCESS_COARSE_LOCATION) == + PackageManager.PERMISSION_GRANTED + if (!fineOk && !coarseOk) { + throw IllegalStateException("LOCATION_PERMISSION_REQUIRED: grant Location permission") + } + val now = System.currentTimeMillis() + val candidates = + providers.mapNotNull { provider -> manager.getLastKnownLocation(provider) } + val freshest = candidates.maxByOrNull { it.time } ?: return null + if (maxAgeMs != null && now - freshest.time > maxAgeMs) return null + return freshest + } + + private suspend fun requestCurrent( + manager: LocationManager, + providers: List, + timeoutMs: Long, + ): Location { + val fineOk = + ContextCompat.checkSelfPermission(context, Manifest.permission.ACCESS_FINE_LOCATION) == + PackageManager.PERMISSION_GRANTED + val coarseOk = + ContextCompat.checkSelfPermission(context, Manifest.permission.ACCESS_COARSE_LOCATION) == + PackageManager.PERMISSION_GRANTED + if (!fineOk && !coarseOk) { + throw IllegalStateException("LOCATION_PERMISSION_REQUIRED: grant Location permission") + } + val resolved = + providers.firstOrNull { manager.isProviderEnabled(it) } + ?: throw IllegalStateException("LOCATION_UNAVAILABLE: no providers available") + return withTimeout(timeoutMs.coerceAtLeast(1)) { + suspendCancellableCoroutine { cont -> + val signal = CancellationSignal() + cont.invokeOnCancellation { signal.cancel() } + manager.getCurrentLocation(resolved, signal, context.mainExecutor) { location -> + if (location != null) { + cont.resume(location) + } else { + cont.resumeWithException(IllegalStateException("LOCATION_UNAVAILABLE: no fix")) + } + } + } + } + } +} diff --git a/apps/android/app/src/main/java/ai/openclaw/android/node/ScreenRecordManager.kt b/apps/android/app/src/main/java/ai/openclaw/android/node/ScreenRecordManager.kt new file mode 100644 index 0000000000000000000000000000000000000000..337a953866a7e43801e4e4f8f80a37d99f4c3571 --- /dev/null +++ b/apps/android/app/src/main/java/ai/openclaw/android/node/ScreenRecordManager.kt @@ -0,0 +1,199 @@ +package ai.openclaw.android.node + +import android.content.Context +import android.hardware.display.DisplayManager +import android.media.MediaRecorder +import android.media.projection.MediaProjectionManager +import android.os.Build +import android.util.Base64 +import ai.openclaw.android.ScreenCaptureRequester +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.delay +import kotlinx.coroutines.withContext +import java.io.File +import kotlin.math.roundToInt + +class ScreenRecordManager(private val context: Context) { + data class Payload(val payloadJson: String) + + @Volatile private var screenCaptureRequester: ScreenCaptureRequester? = null + @Volatile private var permissionRequester: ai.openclaw.android.PermissionRequester? = null + + fun attachScreenCaptureRequester(requester: ScreenCaptureRequester) { + screenCaptureRequester = requester + } + + fun attachPermissionRequester(requester: ai.openclaw.android.PermissionRequester) { + permissionRequester = requester + } + + suspend fun record(paramsJson: String?): Payload = + withContext(Dispatchers.Default) { + val requester = + screenCaptureRequester + ?: throw IllegalStateException( + "SCREEN_PERMISSION_REQUIRED: grant Screen Recording permission", + ) + + val durationMs = (parseDurationMs(paramsJson) ?: 10_000).coerceIn(250, 60_000) + val fps = (parseFps(paramsJson) ?: 10.0).coerceIn(1.0, 60.0) + val fpsInt = fps.roundToInt().coerceIn(1, 60) + val screenIndex = parseScreenIndex(paramsJson) + val includeAudio = parseIncludeAudio(paramsJson) ?: true + val format = parseString(paramsJson, key = "format") + if (format != null && format.lowercase() != "mp4") { + throw IllegalArgumentException("INVALID_REQUEST: screen format must be mp4") + } + if (screenIndex != null && screenIndex != 0) { + throw IllegalArgumentException("INVALID_REQUEST: screenIndex must be 0 on Android") + } + + val capture = requester.requestCapture() + ?: throw IllegalStateException( + "SCREEN_PERMISSION_REQUIRED: grant Screen Recording permission", + ) + + val mgr = + context.getSystemService(Context.MEDIA_PROJECTION_SERVICE) as MediaProjectionManager + val projection = mgr.getMediaProjection(capture.resultCode, capture.data) + ?: throw IllegalStateException("UNAVAILABLE: screen capture unavailable") + + val metrics = context.resources.displayMetrics + val width = metrics.widthPixels + val height = metrics.heightPixels + val densityDpi = metrics.densityDpi + + val file = File.createTempFile("openclaw-screen-", ".mp4") + if (includeAudio) ensureMicPermission() + + val recorder = createMediaRecorder() + var virtualDisplay: android.hardware.display.VirtualDisplay? = null + try { + if (includeAudio) { + recorder.setAudioSource(MediaRecorder.AudioSource.MIC) + } + recorder.setVideoSource(MediaRecorder.VideoSource.SURFACE) + recorder.setOutputFormat(MediaRecorder.OutputFormat.MPEG_4) + recorder.setVideoEncoder(MediaRecorder.VideoEncoder.H264) + if (includeAudio) { + recorder.setAudioEncoder(MediaRecorder.AudioEncoder.AAC) + recorder.setAudioChannels(1) + recorder.setAudioSamplingRate(44_100) + recorder.setAudioEncodingBitRate(96_000) + } + recorder.setVideoSize(width, height) + recorder.setVideoFrameRate(fpsInt) + recorder.setVideoEncodingBitRate(estimateBitrate(width, height, fpsInt)) + recorder.setOutputFile(file.absolutePath) + recorder.prepare() + + val surface = recorder.surface + virtualDisplay = + projection.createVirtualDisplay( + "openclaw-screen", + width, + height, + densityDpi, + DisplayManager.VIRTUAL_DISPLAY_FLAG_AUTO_MIRROR, + surface, + null, + null, + ) + + recorder.start() + delay(durationMs.toLong()) + } finally { + try { + recorder.stop() + } catch (_: Throwable) { + // ignore + } + recorder.reset() + recorder.release() + virtualDisplay?.release() + projection.stop() + } + + val bytes = withContext(Dispatchers.IO) { file.readBytes() } + file.delete() + val base64 = Base64.encodeToString(bytes, Base64.NO_WRAP) + Payload( + """{"format":"mp4","base64":"$base64","durationMs":$durationMs,"fps":$fpsInt,"screenIndex":0,"hasAudio":$includeAudio}""", + ) + } + + private fun createMediaRecorder(): MediaRecorder = MediaRecorder(context) + + private suspend fun ensureMicPermission() { + val granted = + androidx.core.content.ContextCompat.checkSelfPermission( + context, + android.Manifest.permission.RECORD_AUDIO, + ) == android.content.pm.PackageManager.PERMISSION_GRANTED + if (granted) return + + val requester = + permissionRequester + ?: throw IllegalStateException("MIC_PERMISSION_REQUIRED: grant Microphone permission") + val results = requester.requestIfMissing(listOf(android.Manifest.permission.RECORD_AUDIO)) + if (results[android.Manifest.permission.RECORD_AUDIO] != true) { + throw IllegalStateException("MIC_PERMISSION_REQUIRED: grant Microphone permission") + } + } + + private fun parseDurationMs(paramsJson: String?): Int? = + parseNumber(paramsJson, key = "durationMs")?.toIntOrNull() + + private fun parseFps(paramsJson: String?): Double? = + parseNumber(paramsJson, key = "fps")?.toDoubleOrNull() + + private fun parseScreenIndex(paramsJson: String?): Int? = + parseNumber(paramsJson, key = "screenIndex")?.toIntOrNull() + + private fun parseIncludeAudio(paramsJson: String?): Boolean? { + val raw = paramsJson ?: return null + val key = "\"includeAudio\"" + val idx = raw.indexOf(key) + if (idx < 0) return null + val colon = raw.indexOf(':', idx + key.length) + if (colon < 0) return null + val tail = raw.substring(colon + 1).trimStart() + return when { + tail.startsWith("true") -> true + tail.startsWith("false") -> false + else -> null + } + } + + private fun parseNumber(paramsJson: String?, key: String): String? { + val raw = paramsJson ?: return null + val needle = "\"$key\"" + val idx = raw.indexOf(needle) + if (idx < 0) return null + val colon = raw.indexOf(':', idx + needle.length) + if (colon < 0) return null + val tail = raw.substring(colon + 1).trimStart() + return tail.takeWhile { it.isDigit() || it == '.' || it == '-' } + } + + private fun parseString(paramsJson: String?, key: String): String? { + val raw = paramsJson ?: return null + val needle = "\"$key\"" + val idx = raw.indexOf(needle) + if (idx < 0) return null + val colon = raw.indexOf(':', idx + needle.length) + if (colon < 0) return null + val tail = raw.substring(colon + 1).trimStart() + if (!tail.startsWith('\"')) return null + val rest = tail.drop(1) + val end = rest.indexOf('\"') + if (end < 0) return null + return rest.substring(0, end) + } + + private fun estimateBitrate(width: Int, height: Int, fps: Int): Int { + val pixels = width.toLong() * height.toLong() + val raw = (pixels * fps.toLong() * 2L).toInt() + return raw.coerceIn(1_000_000, 12_000_000) + } +} diff --git a/apps/android/app/src/main/java/ai/openclaw/android/node/SmsManager.kt b/apps/android/app/src/main/java/ai/openclaw/android/node/SmsManager.kt new file mode 100644 index 0000000000000000000000000000000000000000..d727bfd2763211d068d8eb245bcb5c6c495ccd99 --- /dev/null +++ b/apps/android/app/src/main/java/ai/openclaw/android/node/SmsManager.kt @@ -0,0 +1,230 @@ +package ai.openclaw.android.node + +import android.Manifest +import android.content.Context +import android.content.pm.PackageManager +import android.telephony.SmsManager as AndroidSmsManager +import androidx.core.content.ContextCompat +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.JsonElement +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.JsonPrimitive +import kotlinx.serialization.json.jsonObject +import kotlinx.serialization.encodeToString +import ai.openclaw.android.PermissionRequester + +/** + * Sends SMS messages via the Android SMS API. + * Requires SEND_SMS permission to be granted. + */ +class SmsManager(private val context: Context) { + + private val json = JsonConfig + @Volatile private var permissionRequester: PermissionRequester? = null + + data class SendResult( + val ok: Boolean, + val to: String, + val message: String?, + val error: String? = null, + val payloadJson: String, + ) + + internal data class ParsedParams( + val to: String, + val message: String, + ) + + internal sealed class ParseResult { + data class Ok(val params: ParsedParams) : ParseResult() + data class Error( + val error: String, + val to: String = "", + val message: String? = null, + ) : ParseResult() + } + + internal data class SendPlan( + val parts: List, + val useMultipart: Boolean, + ) + + companion object { + internal val JsonConfig = Json { ignoreUnknownKeys = true } + + internal fun parseParams(paramsJson: String?, json: Json = JsonConfig): ParseResult { + val params = paramsJson?.trim().orEmpty() + if (params.isEmpty()) { + return ParseResult.Error(error = "INVALID_REQUEST: paramsJSON required") + } + + val obj = try { + json.parseToJsonElement(params).jsonObject + } catch (_: Throwable) { + null + } + + if (obj == null) { + return ParseResult.Error(error = "INVALID_REQUEST: expected JSON object") + } + + val to = (obj["to"] as? JsonPrimitive)?.content?.trim().orEmpty() + val message = (obj["message"] as? JsonPrimitive)?.content.orEmpty() + + if (to.isEmpty()) { + return ParseResult.Error( + error = "INVALID_REQUEST: 'to' phone number required", + message = message, + ) + } + + if (message.isEmpty()) { + return ParseResult.Error( + error = "INVALID_REQUEST: 'message' text required", + to = to, + ) + } + + return ParseResult.Ok(ParsedParams(to = to, message = message)) + } + + internal fun buildSendPlan( + message: String, + divider: (String) -> List, + ): SendPlan { + val parts = divider(message).ifEmpty { listOf(message) } + return SendPlan(parts = parts, useMultipart = parts.size > 1) + } + + internal fun buildPayloadJson( + json: Json = JsonConfig, + ok: Boolean, + to: String, + error: String?, + ): String { + val payload = + mutableMapOf( + "ok" to JsonPrimitive(ok), + "to" to JsonPrimitive(to), + ) + if (!ok) { + payload["error"] = JsonPrimitive(error ?: "SMS_SEND_FAILED") + } + return json.encodeToString(JsonObject.serializer(), JsonObject(payload)) + } + } + + fun hasSmsPermission(): Boolean { + return ContextCompat.checkSelfPermission( + context, + Manifest.permission.SEND_SMS + ) == PackageManager.PERMISSION_GRANTED + } + + fun canSendSms(): Boolean { + return hasSmsPermission() && hasTelephonyFeature() + } + + fun hasTelephonyFeature(): Boolean { + return context.packageManager?.hasSystemFeature(PackageManager.FEATURE_TELEPHONY) == true + } + + fun attachPermissionRequester(requester: PermissionRequester) { + permissionRequester = requester + } + + /** + * Send an SMS message. + * + * @param paramsJson JSON with "to" (phone number) and "message" (text) fields + * @return SendResult indicating success or failure + */ + suspend fun send(paramsJson: String?): SendResult { + if (!hasTelephonyFeature()) { + return errorResult( + error = "SMS_UNAVAILABLE: telephony not available", + ) + } + + if (!ensureSmsPermission()) { + return errorResult( + error = "SMS_PERMISSION_REQUIRED: grant SMS permission", + ) + } + + val parseResult = parseParams(paramsJson, json) + if (parseResult is ParseResult.Error) { + return errorResult( + error = parseResult.error, + to = parseResult.to, + message = parseResult.message, + ) + } + val params = (parseResult as ParseResult.Ok).params + + return try { + val smsManager = context.getSystemService(AndroidSmsManager::class.java) + ?: throw IllegalStateException("SMS_UNAVAILABLE: SmsManager not available") + + val plan = buildSendPlan(params.message) { smsManager.divideMessage(it) } + if (plan.useMultipart) { + smsManager.sendMultipartTextMessage( + params.to, // destination + null, // service center (null = default) + ArrayList(plan.parts), // message parts + null, // sent intents + null, // delivery intents + ) + } else { + smsManager.sendTextMessage( + params.to, // destination + null, // service center (null = default) + params.message,// message + null, // sent intent + null, // delivery intent + ) + } + + okResult(to = params.to, message = params.message) + } catch (e: SecurityException) { + errorResult( + error = "SMS_PERMISSION_REQUIRED: ${e.message}", + to = params.to, + message = params.message, + ) + } catch (e: Throwable) { + errorResult( + error = "SMS_SEND_FAILED: ${e.message ?: "unknown error"}", + to = params.to, + message = params.message, + ) + } + } + + private suspend fun ensureSmsPermission(): Boolean { + if (hasSmsPermission()) return true + val requester = permissionRequester ?: return false + val results = requester.requestIfMissing(listOf(Manifest.permission.SEND_SMS)) + return results[Manifest.permission.SEND_SMS] == true + } + + private fun okResult(to: String, message: String): SendResult { + return SendResult( + ok = true, + to = to, + message = message, + error = null, + payloadJson = buildPayloadJson(json = json, ok = true, to = to, error = null), + ) + } + + private fun errorResult(error: String, to: String = "", message: String? = null): SendResult { + return SendResult( + ok = false, + to = to, + message = message, + error = error, + payloadJson = buildPayloadJson(json = json, ok = false, to = to, error = error), + ) + } +} diff --git a/apps/android/app/src/main/java/ai/openclaw/android/protocol/OpenClawCanvasA2UIAction.kt b/apps/android/app/src/main/java/ai/openclaw/android/protocol/OpenClawCanvasA2UIAction.kt new file mode 100644 index 0000000000000000000000000000000000000000..7e1a5bf127eee84e94388be61a0c10cf4bc8732a --- /dev/null +++ b/apps/android/app/src/main/java/ai/openclaw/android/protocol/OpenClawCanvasA2UIAction.kt @@ -0,0 +1,66 @@ +package ai.openclaw.android.protocol + +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.JsonPrimitive + +object OpenClawCanvasA2UIAction { + fun extractActionName(userAction: JsonObject): String? { + val name = + (userAction["name"] as? JsonPrimitive) + ?.content + ?.trim() + .orEmpty() + if (name.isNotEmpty()) return name + val action = + (userAction["action"] as? JsonPrimitive) + ?.content + ?.trim() + .orEmpty() + return action.ifEmpty { null } + } + + fun sanitizeTagValue(value: String): String { + val trimmed = value.trim().ifEmpty { "-" } + val normalized = trimmed.replace(" ", "_") + val out = StringBuilder(normalized.length) + for (c in normalized) { + val ok = + c.isLetterOrDigit() || + c == '_' || + c == '-' || + c == '.' || + c == ':' + out.append(if (ok) c else '_') + } + return out.toString() + } + + fun formatAgentMessage( + actionName: String, + sessionKey: String, + surfaceId: String, + sourceComponentId: String, + host: String, + instanceId: String, + contextJson: String?, + ): String { + val ctxSuffix = contextJson?.takeIf { it.isNotBlank() }?.let { " ctx=$it" }.orEmpty() + return listOf( + "CANVAS_A2UI", + "action=${sanitizeTagValue(actionName)}", + "session=${sanitizeTagValue(sessionKey)}", + "surface=${sanitizeTagValue(surfaceId)}", + "component=${sanitizeTagValue(sourceComponentId)}", + "host=${sanitizeTagValue(host)}", + "instance=${sanitizeTagValue(instanceId)}$ctxSuffix", + "default=update_canvas", + ).joinToString(separator = " ") + } + + fun jsDispatchA2UIActionStatus(actionId: String, ok: Boolean, error: String?): String { + val err = (error ?: "").replace("\\", "\\\\").replace("\"", "\\\"") + val okLiteral = if (ok) "true" else "false" + val idEscaped = actionId.replace("\\", "\\\\").replace("\"", "\\\"") + return "window.dispatchEvent(new CustomEvent('openclaw:a2ui-action-status', { detail: { id: \"${idEscaped}\", ok: ${okLiteral}, error: \"${err}\" } }));" + } +} diff --git a/apps/android/app/src/main/java/ai/openclaw/android/protocol/OpenClawProtocolConstants.kt b/apps/android/app/src/main/java/ai/openclaw/android/protocol/OpenClawProtocolConstants.kt new file mode 100644 index 0000000000000000000000000000000000000000..ccca40c4c35300086b63a5fd6ba7ccc9a8e1e86e --- /dev/null +++ b/apps/android/app/src/main/java/ai/openclaw/android/protocol/OpenClawProtocolConstants.kt @@ -0,0 +1,71 @@ +package ai.openclaw.android.protocol + +enum class OpenClawCapability(val rawValue: String) { + Canvas("canvas"), + Camera("camera"), + Screen("screen"), + Sms("sms"), + VoiceWake("voiceWake"), + Location("location"), +} + +enum class OpenClawCanvasCommand(val rawValue: String) { + Present("canvas.present"), + Hide("canvas.hide"), + Navigate("canvas.navigate"), + Eval("canvas.eval"), + Snapshot("canvas.snapshot"), + ; + + companion object { + const val NamespacePrefix: String = "canvas." + } +} + +enum class OpenClawCanvasA2UICommand(val rawValue: String) { + Push("canvas.a2ui.push"), + PushJSONL("canvas.a2ui.pushJSONL"), + Reset("canvas.a2ui.reset"), + ; + + companion object { + const val NamespacePrefix: String = "canvas.a2ui." + } +} + +enum class OpenClawCameraCommand(val rawValue: String) { + Snap("camera.snap"), + Clip("camera.clip"), + ; + + companion object { + const val NamespacePrefix: String = "camera." + } +} + +enum class OpenClawScreenCommand(val rawValue: String) { + Record("screen.record"), + ; + + companion object { + const val NamespacePrefix: String = "screen." + } +} + +enum class OpenClawSmsCommand(val rawValue: String) { + Send("sms.send"), + ; + + companion object { + const val NamespacePrefix: String = "sms." + } +} + +enum class OpenClawLocationCommand(val rawValue: String) { + Get("location.get"), + ; + + companion object { + const val NamespacePrefix: String = "location." + } +} diff --git a/apps/android/app/src/main/java/ai/openclaw/android/tools/ToolDisplay.kt b/apps/android/app/src/main/java/ai/openclaw/android/tools/ToolDisplay.kt new file mode 100644 index 0000000000000000000000000000000000000000..1c5561767e632bfa6f41010819827adc5fb5b855 --- /dev/null +++ b/apps/android/app/src/main/java/ai/openclaw/android/tools/ToolDisplay.kt @@ -0,0 +1,222 @@ +package ai.openclaw.android.tools + +import android.content.Context +import kotlinx.serialization.Serializable +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.JsonArray +import kotlinx.serialization.json.JsonElement +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.JsonPrimitive +import kotlinx.serialization.json.contentOrNull + +@Serializable +private data class ToolDisplayActionSpec( + val label: String? = null, + val detailKeys: List? = null, +) + +@Serializable +private data class ToolDisplaySpec( + val emoji: String? = null, + val title: String? = null, + val label: String? = null, + val detailKeys: List? = null, + val actions: Map? = null, +) + +@Serializable +private data class ToolDisplayConfig( + val version: Int? = null, + val fallback: ToolDisplaySpec? = null, + val tools: Map? = null, +) + +data class ToolDisplaySummary( + val name: String, + val emoji: String, + val title: String, + val label: String, + val verb: String?, + val detail: String?, +) { + val detailLine: String? + get() { + val parts = mutableListOf() + if (!verb.isNullOrBlank()) parts.add(verb) + if (!detail.isNullOrBlank()) parts.add(detail) + return if (parts.isEmpty()) null else parts.joinToString(" · ") + } + + val summaryLine: String + get() = if (detailLine != null) "${emoji} ${label}: ${detailLine}" else "${emoji} ${label}" +} + +object ToolDisplayRegistry { + private const val CONFIG_ASSET = "tool-display.json" + + private val json = Json { ignoreUnknownKeys = true } + @Volatile private var cachedConfig: ToolDisplayConfig? = null + + fun resolve( + context: Context, + name: String?, + args: JsonObject?, + meta: String? = null, + ): ToolDisplaySummary { + val trimmedName = name?.trim().orEmpty().ifEmpty { "tool" } + val key = trimmedName.lowercase() + val config = loadConfig(context) + val spec = config.tools?.get(key) + val fallback = config.fallback + + val emoji = spec?.emoji ?: fallback?.emoji ?: "🧩" + val title = spec?.title ?: titleFromName(trimmedName) + val label = spec?.label ?: trimmedName + + val actionRaw = args?.get("action")?.asStringOrNull()?.trim() + val action = actionRaw?.takeIf { it.isNotEmpty() } + val actionSpec = action?.let { spec?.actions?.get(it) } + val verb = normalizeVerb(actionSpec?.label ?: action) + + var detail: String? = null + if (key == "read") { + detail = readDetail(args) + } else if (key == "write" || key == "edit" || key == "attach") { + detail = pathDetail(args) + } + + val detailKeys = actionSpec?.detailKeys ?: spec?.detailKeys ?: fallback?.detailKeys ?: emptyList() + if (detail == null) { + detail = firstValue(args, detailKeys) + } + + if (detail == null) { + detail = meta + } + + if (detail != null) { + detail = shortenHomeInString(detail) + } + + return ToolDisplaySummary( + name = trimmedName, + emoji = emoji, + title = title, + label = label, + verb = verb, + detail = detail, + ) + } + + private fun loadConfig(context: Context): ToolDisplayConfig { + val existing = cachedConfig + if (existing != null) return existing + return try { + val jsonString = context.assets.open(CONFIG_ASSET).bufferedReader().use { it.readText() } + val decoded = json.decodeFromString(ToolDisplayConfig.serializer(), jsonString) + cachedConfig = decoded + decoded + } catch (_: Throwable) { + val fallback = ToolDisplayConfig() + cachedConfig = fallback + fallback + } + } + + private fun titleFromName(name: String): String { + val cleaned = name.replace("_", " ").trim() + if (cleaned.isEmpty()) return "Tool" + return cleaned + .split(Regex("\\s+")) + .joinToString(" ") { part -> + val upper = part.uppercase() + if (part.length <= 2 && part == upper) part + else upper.firstOrNull()?.toString().orEmpty() + part.lowercase().drop(1) + } + } + + private fun normalizeVerb(value: String?): String? { + val trimmed = value?.trim().orEmpty() + if (trimmed.isEmpty()) return null + return trimmed.replace("_", " ") + } + + private fun readDetail(args: JsonObject?): String? { + val path = args?.get("path")?.asStringOrNull() ?: return null + val offset = args["offset"].asNumberOrNull() + val limit = args["limit"].asNumberOrNull() + return if (offset != null && limit != null) { + val end = offset + limit + "${path}:${offset.toInt()}-${end.toInt()}" + } else { + path + } + } + + private fun pathDetail(args: JsonObject?): String? { + return args?.get("path")?.asStringOrNull() + } + + private fun firstValue(args: JsonObject?, keys: List): String? { + for (key in keys) { + val value = valueForPath(args, key) + val rendered = renderValue(value) + if (!rendered.isNullOrBlank()) return rendered + } + return null + } + + private fun valueForPath(args: JsonObject?, path: String): JsonElement? { + var current: JsonElement? = args + for (segment in path.split(".")) { + if (segment.isBlank()) return null + val obj = current as? JsonObject ?: return null + current = obj[segment] + } + return current + } + + private fun renderValue(value: JsonElement?): String? { + if (value == null) return null + if (value is JsonPrimitive) { + if (value.isString) { + val trimmed = value.contentOrNull?.trim().orEmpty() + if (trimmed.isEmpty()) return null + val firstLine = trimmed.lineSequence().firstOrNull()?.trim().orEmpty() + if (firstLine.isEmpty()) return null + return if (firstLine.length > 160) "${firstLine.take(157)}…" else firstLine + } + val raw = value.contentOrNull?.trim().orEmpty() + raw.toBooleanStrictOrNull()?.let { return it.toString() } + raw.toLongOrNull()?.let { return it.toString() } + raw.toDoubleOrNull()?.let { return it.toString() } + } + if (value is JsonArray) { + val items = value.mapNotNull { renderValue(it) } + if (items.isEmpty()) return null + val preview = items.take(3).joinToString(", ") + return if (items.size > 3) "${preview}…" else preview + } + return null + } + + private fun shortenHomeInString(value: String): String { + val home = System.getProperty("user.home")?.takeIf { it.isNotBlank() } + ?: System.getenv("HOME")?.takeIf { it.isNotBlank() } + if (home.isNullOrEmpty()) return value + return value.replace(home, "~") + .replace(Regex("/Users/[^/]+"), "~") + .replace(Regex("/home/[^/]+"), "~") + } + + private fun JsonElement?.asStringOrNull(): String? { + val primitive = this as? JsonPrimitive ?: return null + return if (primitive.isString) primitive.contentOrNull else primitive.toString() + } + + private fun JsonElement?.asNumberOrNull(): Double? { + val primitive = this as? JsonPrimitive ?: return null + val raw = primitive.contentOrNull ?: return null + return raw.toDoubleOrNull() + } +} diff --git a/apps/android/app/src/main/java/ai/openclaw/android/ui/CameraHudOverlay.kt b/apps/android/app/src/main/java/ai/openclaw/android/ui/CameraHudOverlay.kt new file mode 100644 index 0000000000000000000000000000000000000000..21043d739b0b9e6d00a4356c2002c7201cb3ab92 --- /dev/null +++ b/apps/android/app/src/main/java/ai/openclaw/android/ui/CameraHudOverlay.kt @@ -0,0 +1,44 @@ +package ai.openclaw.android.ui + +import androidx.compose.foundation.background +import androidx.compose.foundation.layout.Box +import androidx.compose.foundation.layout.fillMaxSize +import androidx.compose.runtime.Composable +import androidx.compose.runtime.LaunchedEffect +import androidx.compose.runtime.getValue +import androidx.compose.runtime.mutableFloatStateOf +import androidx.compose.runtime.remember +import androidx.compose.runtime.setValue +import androidx.compose.ui.Modifier +import androidx.compose.ui.draw.alpha +import androidx.compose.ui.graphics.Color +import kotlinx.coroutines.delay + +@Composable +fun CameraFlashOverlay( + token: Long, + modifier: Modifier = Modifier, +) { + Box(modifier = modifier.fillMaxSize()) { + CameraFlash(token = token) + } +} + +@Composable +private fun CameraFlash(token: Long) { + var alpha by remember { mutableFloatStateOf(0f) } + LaunchedEffect(token) { + if (token == 0L) return@LaunchedEffect + alpha = 0.85f + delay(110) + alpha = 0f + } + + Box( + modifier = + Modifier + .fillMaxSize() + .alpha(alpha) + .background(Color.White), + ) +} diff --git a/apps/android/app/src/main/java/ai/openclaw/android/ui/ChatSheet.kt b/apps/android/app/src/main/java/ai/openclaw/android/ui/ChatSheet.kt new file mode 100644 index 0000000000000000000000000000000000000000..85f20364c61660bd99a8daedbe215045f7f12472 --- /dev/null +++ b/apps/android/app/src/main/java/ai/openclaw/android/ui/ChatSheet.kt @@ -0,0 +1,10 @@ +package ai.openclaw.android.ui + +import androidx.compose.runtime.Composable +import ai.openclaw.android.MainViewModel +import ai.openclaw.android.ui.chat.ChatSheetContent + +@Composable +fun ChatSheet(viewModel: MainViewModel) { + ChatSheetContent(viewModel = viewModel) +} diff --git a/apps/android/app/src/main/java/ai/openclaw/android/ui/OpenClawTheme.kt b/apps/android/app/src/main/java/ai/openclaw/android/ui/OpenClawTheme.kt new file mode 100644 index 0000000000000000000000000000000000000000..aad743a6d7d540223ce06cfd136fc14e6f2472f3 --- /dev/null +++ b/apps/android/app/src/main/java/ai/openclaw/android/ui/OpenClawTheme.kt @@ -0,0 +1,32 @@ +package ai.openclaw.android.ui + +import androidx.compose.foundation.isSystemInDarkTheme +import androidx.compose.material3.MaterialTheme +import androidx.compose.material3.dynamicDarkColorScheme +import androidx.compose.material3.dynamicLightColorScheme +import androidx.compose.runtime.Composable +import androidx.compose.ui.graphics.Color +import androidx.compose.ui.platform.LocalContext + +@Composable +fun OpenClawTheme(content: @Composable () -> Unit) { + val context = LocalContext.current + val isDark = isSystemInDarkTheme() + val colorScheme = if (isDark) dynamicDarkColorScheme(context) else dynamicLightColorScheme(context) + + MaterialTheme(colorScheme = colorScheme, content = content) +} + +@Composable +fun overlayContainerColor(): Color { + val scheme = MaterialTheme.colorScheme + val isDark = isSystemInDarkTheme() + val base = if (isDark) scheme.surfaceContainerLow else scheme.surfaceContainerHigh + // Light mode: background stays dark (canvas), so clamp overlays away from pure-white glare. + return if (isDark) base else base.copy(alpha = 0.88f) +} + +@Composable +fun overlayIconColor(): Color { + return MaterialTheme.colorScheme.onSurfaceVariant +} diff --git a/apps/android/app/src/main/java/ai/openclaw/android/ui/RootScreen.kt b/apps/android/app/src/main/java/ai/openclaw/android/ui/RootScreen.kt new file mode 100644 index 0000000000000000000000000000000000000000..af0cfe628ac07f7e7c20dee847a5469f044ae829 --- /dev/null +++ b/apps/android/app/src/main/java/ai/openclaw/android/ui/RootScreen.kt @@ -0,0 +1,429 @@ +package ai.openclaw.android.ui + +import android.annotation.SuppressLint +import android.Manifest +import android.content.pm.PackageManager +import android.graphics.Color +import android.util.Log +import android.view.View +import android.webkit.JavascriptInterface +import android.webkit.ConsoleMessage +import android.webkit.WebChromeClient +import android.webkit.WebView +import android.webkit.WebSettings +import android.webkit.WebResourceError +import android.webkit.WebResourceRequest +import android.webkit.WebResourceResponse +import android.webkit.WebViewClient +import androidx.activity.compose.rememberLauncherForActivityResult +import androidx.activity.result.contract.ActivityResultContracts +import androidx.webkit.WebSettingsCompat +import androidx.webkit.WebViewFeature +import androidx.compose.foundation.layout.Arrangement +import androidx.compose.foundation.layout.Box +import androidx.compose.foundation.layout.Column +import androidx.compose.foundation.layout.WindowInsets +import androidx.compose.foundation.layout.WindowInsetsSides +import androidx.compose.foundation.layout.only +import androidx.compose.foundation.layout.fillMaxSize +import androidx.compose.foundation.layout.padding +import androidx.compose.foundation.layout.safeDrawing +import androidx.compose.foundation.layout.size +import androidx.compose.foundation.layout.windowInsetsPadding +import androidx.compose.material3.ExperimentalMaterial3Api +import androidx.compose.material3.FilledTonalIconButton +import androidx.compose.material3.Icon +import androidx.compose.material3.IconButtonDefaults +import androidx.compose.material3.LocalContentColor +import androidx.compose.material3.MaterialTheme +import androidx.compose.material3.ModalBottomSheet +import androidx.compose.material3.rememberModalBottomSheetState +import androidx.compose.material.icons.Icons +import androidx.compose.material.icons.automirrored.filled.ScreenShare +import androidx.compose.material.icons.filled.ChatBubble +import androidx.compose.material.icons.filled.CheckCircle +import androidx.compose.material.icons.filled.Error +import androidx.compose.material.icons.filled.FiberManualRecord +import androidx.compose.material.icons.filled.PhotoCamera +import androidx.compose.material.icons.filled.RecordVoiceOver +import androidx.compose.material.icons.filled.Refresh +import androidx.compose.material.icons.filled.Report +import androidx.compose.material.icons.filled.Settings +import androidx.compose.runtime.Composable +import androidx.compose.runtime.collectAsState +import androidx.compose.runtime.getValue +import androidx.compose.runtime.mutableStateOf +import androidx.compose.runtime.remember +import androidx.compose.runtime.setValue +import androidx.compose.ui.Alignment +import androidx.compose.ui.Modifier +import androidx.compose.ui.graphics.Color as ComposeColor +import androidx.compose.ui.graphics.lerp +import androidx.compose.ui.platform.LocalContext +import androidx.compose.ui.unit.dp +import androidx.compose.ui.viewinterop.AndroidView +import androidx.compose.ui.window.Popup +import androidx.compose.ui.window.PopupProperties +import androidx.core.content.ContextCompat +import ai.openclaw.android.CameraHudKind +import ai.openclaw.android.MainViewModel + +@OptIn(ExperimentalMaterial3Api::class) +@Composable +fun RootScreen(viewModel: MainViewModel) { + var sheet by remember { mutableStateOf(null) } + val sheetState = rememberModalBottomSheetState(skipPartiallyExpanded = true) + val safeOverlayInsets = WindowInsets.safeDrawing.only(WindowInsetsSides.Top + WindowInsetsSides.Horizontal) + val context = LocalContext.current + val serverName by viewModel.serverName.collectAsState() + val statusText by viewModel.statusText.collectAsState() + val cameraHud by viewModel.cameraHud.collectAsState() + val cameraFlashToken by viewModel.cameraFlashToken.collectAsState() + val screenRecordActive by viewModel.screenRecordActive.collectAsState() + val isForeground by viewModel.isForeground.collectAsState() + val voiceWakeStatusText by viewModel.voiceWakeStatusText.collectAsState() + val talkEnabled by viewModel.talkEnabled.collectAsState() + val talkStatusText by viewModel.talkStatusText.collectAsState() + val talkIsListening by viewModel.talkIsListening.collectAsState() + val talkIsSpeaking by viewModel.talkIsSpeaking.collectAsState() + val seamColorArgb by viewModel.seamColorArgb.collectAsState() + val seamColor = remember(seamColorArgb) { ComposeColor(seamColorArgb) } + val audioPermissionLauncher = + rememberLauncherForActivityResult(ActivityResultContracts.RequestPermission()) { granted -> + if (granted) viewModel.setTalkEnabled(true) + } + val activity = + remember(cameraHud, screenRecordActive, isForeground, statusText, voiceWakeStatusText) { + // Status pill owns transient activity state so it doesn't overlap the connection indicator. + if (!isForeground) { + return@remember StatusActivity( + title = "Foreground required", + icon = Icons.Default.Report, + contentDescription = "Foreground required", + ) + } + + val lowerStatus = statusText.lowercase() + if (lowerStatus.contains("repair")) { + return@remember StatusActivity( + title = "Repairing…", + icon = Icons.Default.Refresh, + contentDescription = "Repairing", + ) + } + if (lowerStatus.contains("pairing") || lowerStatus.contains("approval")) { + return@remember StatusActivity( + title = "Approval pending", + icon = Icons.Default.RecordVoiceOver, + contentDescription = "Approval pending", + ) + } + // Avoid duplicating the primary gateway status ("Connecting…") in the activity slot. + + if (screenRecordActive) { + return@remember StatusActivity( + title = "Recording screen…", + icon = Icons.AutoMirrored.Filled.ScreenShare, + contentDescription = "Recording screen", + tint = androidx.compose.ui.graphics.Color.Red, + ) + } + + cameraHud?.let { hud -> + return@remember when (hud.kind) { + CameraHudKind.Photo -> + StatusActivity( + title = hud.message, + icon = Icons.Default.PhotoCamera, + contentDescription = "Taking photo", + ) + CameraHudKind.Recording -> + StatusActivity( + title = hud.message, + icon = Icons.Default.FiberManualRecord, + contentDescription = "Recording", + tint = androidx.compose.ui.graphics.Color.Red, + ) + CameraHudKind.Success -> + StatusActivity( + title = hud.message, + icon = Icons.Default.CheckCircle, + contentDescription = "Capture finished", + ) + CameraHudKind.Error -> + StatusActivity( + title = hud.message, + icon = Icons.Default.Error, + contentDescription = "Capture failed", + tint = androidx.compose.ui.graphics.Color.Red, + ) + } + } + + if (voiceWakeStatusText.contains("Microphone permission", ignoreCase = true)) { + return@remember StatusActivity( + title = "Mic permission", + icon = Icons.Default.Error, + contentDescription = "Mic permission required", + ) + } + if (voiceWakeStatusText == "Paused") { + val suffix = if (!isForeground) " (background)" else "" + return@remember StatusActivity( + title = "Voice Wake paused$suffix", + icon = Icons.Default.RecordVoiceOver, + contentDescription = "Voice Wake paused", + ) + } + + null + } + + val gatewayState = + remember(serverName, statusText) { + when { + serverName != null -> GatewayState.Connected + statusText.contains("connecting", ignoreCase = true) || + statusText.contains("reconnecting", ignoreCase = true) -> GatewayState.Connecting + statusText.contains("error", ignoreCase = true) -> GatewayState.Error + else -> GatewayState.Disconnected + } + } + + val voiceEnabled = + ContextCompat.checkSelfPermission(context, Manifest.permission.RECORD_AUDIO) == + PackageManager.PERMISSION_GRANTED + + Box(modifier = Modifier.fillMaxSize()) { + CanvasView(viewModel = viewModel, modifier = Modifier.fillMaxSize()) + } + + // Camera flash must be in a Popup to render above the WebView. + Popup(alignment = Alignment.Center, properties = PopupProperties(focusable = false)) { + CameraFlashOverlay(token = cameraFlashToken, modifier = Modifier.fillMaxSize()) + } + + // Keep the overlay buttons above the WebView canvas (AndroidView), otherwise they may not receive touches. + Popup(alignment = Alignment.TopStart, properties = PopupProperties(focusable = false)) { + StatusPill( + gateway = gatewayState, + voiceEnabled = voiceEnabled, + activity = activity, + onClick = { sheet = Sheet.Settings }, + modifier = Modifier.windowInsetsPadding(safeOverlayInsets).padding(start = 12.dp, top = 12.dp), + ) + } + + Popup(alignment = Alignment.TopEnd, properties = PopupProperties(focusable = false)) { + Column( + modifier = Modifier.windowInsetsPadding(safeOverlayInsets).padding(end = 12.dp, top = 12.dp), + verticalArrangement = Arrangement.spacedBy(10.dp), + horizontalAlignment = Alignment.End, + ) { + OverlayIconButton( + onClick = { sheet = Sheet.Chat }, + icon = { Icon(Icons.Default.ChatBubble, contentDescription = "Chat") }, + ) + + // Talk mode gets a dedicated side bubble instead of burying it in settings. + val baseOverlay = overlayContainerColor() + val talkContainer = + lerp( + baseOverlay, + seamColor.copy(alpha = baseOverlay.alpha), + if (talkEnabled) 0.35f else 0.22f, + ) + val talkContent = if (talkEnabled) seamColor else overlayIconColor() + OverlayIconButton( + onClick = { + val next = !talkEnabled + if (next) { + val micOk = + ContextCompat.checkSelfPermission(context, Manifest.permission.RECORD_AUDIO) == + PackageManager.PERMISSION_GRANTED + if (!micOk) audioPermissionLauncher.launch(Manifest.permission.RECORD_AUDIO) + viewModel.setTalkEnabled(true) + } else { + viewModel.setTalkEnabled(false) + } + }, + containerColor = talkContainer, + contentColor = talkContent, + icon = { + Icon( + Icons.Default.RecordVoiceOver, + contentDescription = "Talk Mode", + ) + }, + ) + + OverlayIconButton( + onClick = { sheet = Sheet.Settings }, + icon = { Icon(Icons.Default.Settings, contentDescription = "Settings") }, + ) + } + } + + if (talkEnabled) { + Popup(alignment = Alignment.Center, properties = PopupProperties(focusable = false)) { + TalkOrbOverlay( + seamColor = seamColor, + statusText = talkStatusText, + isListening = talkIsListening, + isSpeaking = talkIsSpeaking, + ) + } + } + + val currentSheet = sheet + if (currentSheet != null) { + ModalBottomSheet( + onDismissRequest = { sheet = null }, + sheetState = sheetState, + ) { + when (currentSheet) { + Sheet.Chat -> ChatSheet(viewModel = viewModel) + Sheet.Settings -> SettingsSheet(viewModel = viewModel) + } + } + } +} + +private enum class Sheet { + Chat, + Settings, +} + +@Composable +private fun OverlayIconButton( + onClick: () -> Unit, + icon: @Composable () -> Unit, + containerColor: ComposeColor? = null, + contentColor: ComposeColor? = null, +) { + FilledTonalIconButton( + onClick = onClick, + modifier = Modifier.size(44.dp), + colors = + IconButtonDefaults.filledTonalIconButtonColors( + containerColor = containerColor ?: overlayContainerColor(), + contentColor = contentColor ?: overlayIconColor(), + ), + ) { + icon() + } +} + +@SuppressLint("SetJavaScriptEnabled") +@Composable +private fun CanvasView(viewModel: MainViewModel, modifier: Modifier = Modifier) { + val context = LocalContext.current + val isDebuggable = (context.applicationInfo.flags and android.content.pm.ApplicationInfo.FLAG_DEBUGGABLE) != 0 + AndroidView( + modifier = modifier, + factory = { + WebView(context).apply { + settings.javaScriptEnabled = true + // Some embedded web UIs (incl. the "background website") use localStorage/sessionStorage. + settings.domStorageEnabled = true + settings.mixedContentMode = WebSettings.MIXED_CONTENT_COMPATIBILITY_MODE + if (WebViewFeature.isFeatureSupported(WebViewFeature.ALGORITHMIC_DARKENING)) { + WebSettingsCompat.setAlgorithmicDarkeningAllowed(settings, false) + } else { + disableForceDarkIfSupported(settings) + } + if (isDebuggable) { + Log.d("OpenClawWebView", "userAgent: ${settings.userAgentString}") + } + isScrollContainer = true + overScrollMode = View.OVER_SCROLL_IF_CONTENT_SCROLLS + isVerticalScrollBarEnabled = true + isHorizontalScrollBarEnabled = true + webViewClient = + object : WebViewClient() { + override fun onReceivedError( + view: WebView, + request: WebResourceRequest, + error: WebResourceError, + ) { + if (!isDebuggable) return + if (!request.isForMainFrame) return + Log.e("OpenClawWebView", "onReceivedError: ${error.errorCode} ${error.description} ${request.url}") + } + + override fun onReceivedHttpError( + view: WebView, + request: WebResourceRequest, + errorResponse: WebResourceResponse, + ) { + if (!isDebuggable) return + if (!request.isForMainFrame) return + Log.e( + "OpenClawWebView", + "onReceivedHttpError: ${errorResponse.statusCode} ${errorResponse.reasonPhrase} ${request.url}", + ) + } + + override fun onPageFinished(view: WebView, url: String?) { + if (isDebuggable) { + Log.d("OpenClawWebView", "onPageFinished: $url") + } + viewModel.canvas.onPageFinished() + } + + override fun onRenderProcessGone( + view: WebView, + detail: android.webkit.RenderProcessGoneDetail, + ): Boolean { + if (isDebuggable) { + Log.e( + "OpenClawWebView", + "onRenderProcessGone didCrash=${detail.didCrash()} priorityAtExit=${detail.rendererPriorityAtExit()}", + ) + } + return true + } + } + webChromeClient = + object : WebChromeClient() { + override fun onConsoleMessage(consoleMessage: ConsoleMessage?): Boolean { + if (!isDebuggable) return false + val msg = consoleMessage ?: return false + Log.d( + "OpenClawWebView", + "console ${msg.messageLevel()} @ ${msg.sourceId()}:${msg.lineNumber()} ${msg.message()}", + ) + return false + } + } + // Use default layer/background; avoid forcing a black fill over WebView content. + + val a2uiBridge = + CanvasA2UIActionBridge { payload -> + viewModel.handleCanvasA2UIActionFromWebView(payload) + } + addJavascriptInterface(a2uiBridge, CanvasA2UIActionBridge.interfaceName) + viewModel.canvas.attach(this) + } + }, + ) +} + +private fun disableForceDarkIfSupported(settings: WebSettings) { + if (!WebViewFeature.isFeatureSupported(WebViewFeature.FORCE_DARK)) return + @Suppress("DEPRECATION") + WebSettingsCompat.setForceDark(settings, WebSettingsCompat.FORCE_DARK_OFF) +} + +private class CanvasA2UIActionBridge(private val onMessage: (String) -> Unit) { + @JavascriptInterface + fun postMessage(payload: String?) { + val msg = payload?.trim().orEmpty() + if (msg.isEmpty()) return + onMessage(msg) + } + + companion object { + const val interfaceName: String = "openclawCanvasA2UIAction" + } +} diff --git a/apps/android/app/src/main/java/ai/openclaw/android/ui/SettingsSheet.kt b/apps/android/app/src/main/java/ai/openclaw/android/ui/SettingsSheet.kt new file mode 100644 index 0000000000000000000000000000000000000000..fa32f7bb85224245981f7d1fb0d4c9ac687881a8 --- /dev/null +++ b/apps/android/app/src/main/java/ai/openclaw/android/ui/SettingsSheet.kt @@ -0,0 +1,686 @@ +package ai.openclaw.android.ui + +import android.Manifest +import android.content.Context +import android.content.Intent +import android.content.pm.PackageManager +import android.net.Uri +import android.os.Build +import android.provider.Settings +import androidx.activity.compose.rememberLauncherForActivityResult +import androidx.activity.result.contract.ActivityResultContracts +import androidx.compose.animation.AnimatedVisibility +import androidx.compose.foundation.clickable +import androidx.compose.foundation.layout.Arrangement +import androidx.compose.foundation.layout.Column +import androidx.compose.foundation.layout.PaddingValues +import androidx.compose.foundation.layout.Row +import androidx.compose.foundation.layout.Spacer +import androidx.compose.foundation.layout.WindowInsets +import androidx.compose.foundation.layout.WindowInsetsSides +import androidx.compose.foundation.layout.fillMaxHeight +import androidx.compose.foundation.layout.fillMaxWidth +import androidx.compose.foundation.layout.height +import androidx.compose.foundation.layout.imePadding +import androidx.compose.foundation.layout.only +import androidx.compose.foundation.layout.safeDrawing +import androidx.compose.foundation.layout.windowInsetsPadding +import androidx.compose.foundation.lazy.LazyColumn +import androidx.compose.foundation.lazy.items +import androidx.compose.foundation.lazy.rememberLazyListState +import androidx.compose.foundation.text.KeyboardActions +import androidx.compose.foundation.text.KeyboardOptions +import androidx.compose.material.icons.Icons +import androidx.compose.material.icons.filled.ExpandLess +import androidx.compose.material.icons.filled.ExpandMore +import androidx.compose.material3.Button +import androidx.compose.material3.HorizontalDivider +import androidx.compose.material3.Icon +import androidx.compose.material3.ListItem +import androidx.compose.material3.MaterialTheme +import androidx.compose.material3.OutlinedTextField +import androidx.compose.material3.RadioButton +import androidx.compose.material3.Switch +import androidx.compose.material3.Text +import androidx.compose.runtime.Composable +import androidx.compose.runtime.LaunchedEffect +import androidx.compose.runtime.collectAsState +import androidx.compose.runtime.getValue +import androidx.compose.runtime.mutableStateOf +import androidx.compose.runtime.remember +import androidx.compose.runtime.setValue +import androidx.compose.ui.Modifier +import androidx.compose.ui.draw.alpha +import androidx.compose.ui.focus.onFocusChanged +import androidx.compose.ui.platform.LocalContext +import androidx.compose.ui.platform.LocalFocusManager +import androidx.compose.ui.text.input.ImeAction +import androidx.compose.ui.text.style.TextAlign +import androidx.compose.ui.unit.dp +import androidx.core.content.ContextCompat +import ai.openclaw.android.BuildConfig +import ai.openclaw.android.LocationMode +import ai.openclaw.android.MainViewModel +import ai.openclaw.android.NodeForegroundService +import ai.openclaw.android.VoiceWakeMode +import ai.openclaw.android.WakeWords + +@Composable +fun SettingsSheet(viewModel: MainViewModel) { + val context = LocalContext.current + val instanceId by viewModel.instanceId.collectAsState() + val displayName by viewModel.displayName.collectAsState() + val cameraEnabled by viewModel.cameraEnabled.collectAsState() + val locationMode by viewModel.locationMode.collectAsState() + val locationPreciseEnabled by viewModel.locationPreciseEnabled.collectAsState() + val preventSleep by viewModel.preventSleep.collectAsState() + val wakeWords by viewModel.wakeWords.collectAsState() + val voiceWakeMode by viewModel.voiceWakeMode.collectAsState() + val voiceWakeStatusText by viewModel.voiceWakeStatusText.collectAsState() + val isConnected by viewModel.isConnected.collectAsState() + val manualEnabled by viewModel.manualEnabled.collectAsState() + val manualHost by viewModel.manualHost.collectAsState() + val manualPort by viewModel.manualPort.collectAsState() + val manualTls by viewModel.manualTls.collectAsState() + val canvasDebugStatusEnabled by viewModel.canvasDebugStatusEnabled.collectAsState() + val statusText by viewModel.statusText.collectAsState() + val serverName by viewModel.serverName.collectAsState() + val remoteAddress by viewModel.remoteAddress.collectAsState() + val gateways by viewModel.gateways.collectAsState() + val discoveryStatusText by viewModel.discoveryStatusText.collectAsState() + + val listState = rememberLazyListState() + val (wakeWordsText, setWakeWordsText) = remember { mutableStateOf("") } + val (advancedExpanded, setAdvancedExpanded) = remember { mutableStateOf(false) } + val focusManager = LocalFocusManager.current + var wakeWordsHadFocus by remember { mutableStateOf(false) } + val deviceModel = + remember { + listOfNotNull(Build.MANUFACTURER, Build.MODEL) + .joinToString(" ") + .trim() + .ifEmpty { "Android" } + } + val appVersion = + remember { + val versionName = BuildConfig.VERSION_NAME.trim().ifEmpty { "dev" } + if (BuildConfig.DEBUG && !versionName.contains("dev", ignoreCase = true)) { + "$versionName-dev" + } else { + versionName + } + } + + LaunchedEffect(wakeWords) { setWakeWordsText(wakeWords.joinToString(", ")) } + val commitWakeWords = { + val parsed = WakeWords.parseIfChanged(wakeWordsText, wakeWords) + if (parsed != null) { + viewModel.setWakeWords(parsed) + } + } + + val permissionLauncher = + rememberLauncherForActivityResult(ActivityResultContracts.RequestMultiplePermissions()) { perms -> + val cameraOk = perms[Manifest.permission.CAMERA] == true + viewModel.setCameraEnabled(cameraOk) + } + + var pendingLocationMode by remember { mutableStateOf(null) } + var pendingPreciseToggle by remember { mutableStateOf(false) } + + val locationPermissionLauncher = + rememberLauncherForActivityResult(ActivityResultContracts.RequestMultiplePermissions()) { perms -> + val fineOk = perms[Manifest.permission.ACCESS_FINE_LOCATION] == true + val coarseOk = perms[Manifest.permission.ACCESS_COARSE_LOCATION] == true + val granted = fineOk || coarseOk + val requestedMode = pendingLocationMode + pendingLocationMode = null + + if (pendingPreciseToggle) { + pendingPreciseToggle = false + viewModel.setLocationPreciseEnabled(fineOk) + return@rememberLauncherForActivityResult + } + + if (!granted) { + viewModel.setLocationMode(LocationMode.Off) + return@rememberLauncherForActivityResult + } + + if (requestedMode != null) { + viewModel.setLocationMode(requestedMode) + if (requestedMode == LocationMode.Always) { + val backgroundOk = + ContextCompat.checkSelfPermission(context, Manifest.permission.ACCESS_BACKGROUND_LOCATION) == + PackageManager.PERMISSION_GRANTED + if (!backgroundOk) { + openAppSettings(context) + } + } + } + } + + val audioPermissionLauncher = + rememberLauncherForActivityResult(ActivityResultContracts.RequestPermission()) { _ -> + // Status text is handled by NodeRuntime. + } + + val smsPermissionAvailable = + remember { + context.packageManager?.hasSystemFeature(PackageManager.FEATURE_TELEPHONY) == true + } + var smsPermissionGranted by + remember { + mutableStateOf( + ContextCompat.checkSelfPermission(context, Manifest.permission.SEND_SMS) == + PackageManager.PERMISSION_GRANTED, + ) + } + val smsPermissionLauncher = + rememberLauncherForActivityResult(ActivityResultContracts.RequestPermission()) { granted -> + smsPermissionGranted = granted + viewModel.refreshGatewayConnection() + } + + fun setCameraEnabledChecked(checked: Boolean) { + if (!checked) { + viewModel.setCameraEnabled(false) + return + } + + val cameraOk = + ContextCompat.checkSelfPermission(context, Manifest.permission.CAMERA) == + PackageManager.PERMISSION_GRANTED + if (cameraOk) { + viewModel.setCameraEnabled(true) + } else { + permissionLauncher.launch(arrayOf(Manifest.permission.CAMERA, Manifest.permission.RECORD_AUDIO)) + } + } + + fun requestLocationPermissions(targetMode: LocationMode) { + val fineOk = + ContextCompat.checkSelfPermission(context, Manifest.permission.ACCESS_FINE_LOCATION) == + PackageManager.PERMISSION_GRANTED + val coarseOk = + ContextCompat.checkSelfPermission(context, Manifest.permission.ACCESS_COARSE_LOCATION) == + PackageManager.PERMISSION_GRANTED + if (fineOk || coarseOk) { + viewModel.setLocationMode(targetMode) + if (targetMode == LocationMode.Always) { + val backgroundOk = + ContextCompat.checkSelfPermission(context, Manifest.permission.ACCESS_BACKGROUND_LOCATION) == + PackageManager.PERMISSION_GRANTED + if (!backgroundOk) { + openAppSettings(context) + } + } + } else { + pendingLocationMode = targetMode + locationPermissionLauncher.launch( + arrayOf(Manifest.permission.ACCESS_FINE_LOCATION, Manifest.permission.ACCESS_COARSE_LOCATION), + ) + } + } + + fun setPreciseLocationChecked(checked: Boolean) { + if (!checked) { + viewModel.setLocationPreciseEnabled(false) + return + } + val fineOk = + ContextCompat.checkSelfPermission(context, Manifest.permission.ACCESS_FINE_LOCATION) == + PackageManager.PERMISSION_GRANTED + if (fineOk) { + viewModel.setLocationPreciseEnabled(true) + } else { + pendingPreciseToggle = true + locationPermissionLauncher.launch(arrayOf(Manifest.permission.ACCESS_FINE_LOCATION)) + } + } + + val visibleGateways = + if (isConnected && remoteAddress != null) { + gateways.filterNot { "${it.host}:${it.port}" == remoteAddress } + } else { + gateways + } + + val gatewayDiscoveryFooterText = + if (visibleGateways.isEmpty()) { + discoveryStatusText + } else if (isConnected) { + "Discovery active • ${visibleGateways.size} other gateway${if (visibleGateways.size == 1) "" else "s"} found" + } else { + "Discovery active • ${visibleGateways.size} gateway${if (visibleGateways.size == 1) "" else "s"} found" + } + + LazyColumn( + state = listState, + modifier = + Modifier + .fillMaxWidth() + .fillMaxHeight() + .imePadding() + .windowInsetsPadding(WindowInsets.safeDrawing.only(WindowInsetsSides.Bottom)), + contentPadding = PaddingValues(16.dp), + verticalArrangement = Arrangement.spacedBy(6.dp), + ) { + // Order parity: Node → Gateway → Voice → Camera → Messaging → Location → Screen. + item { Text("Node", style = MaterialTheme.typography.titleSmall) } + item { + OutlinedTextField( + value = displayName, + onValueChange = viewModel::setDisplayName, + label = { Text("Name") }, + modifier = Modifier.fillMaxWidth(), + ) + } + item { Text("Instance ID: $instanceId", color = MaterialTheme.colorScheme.onSurfaceVariant) } + item { Text("Device: $deviceModel", color = MaterialTheme.colorScheme.onSurfaceVariant) } + item { Text("Version: $appVersion", color = MaterialTheme.colorScheme.onSurfaceVariant) } + + item { HorizontalDivider() } + + // Gateway + item { Text("Gateway", style = MaterialTheme.typography.titleSmall) } + item { ListItem(headlineContent = { Text("Status") }, supportingContent = { Text(statusText) }) } + if (serverName != null) { + item { ListItem(headlineContent = { Text("Server") }, supportingContent = { Text(serverName!!) }) } + } + if (remoteAddress != null) { + item { ListItem(headlineContent = { Text("Address") }, supportingContent = { Text(remoteAddress!!) }) } + } + item { + // UI sanity: "Disconnect" only when we have an active remote. + if (isConnected && remoteAddress != null) { + Button( + onClick = { + viewModel.disconnect() + NodeForegroundService.stop(context) + }, + ) { + Text("Disconnect") + } + } + } + + item { HorizontalDivider() } + + if (!isConnected || visibleGateways.isNotEmpty()) { + item { + Text( + if (isConnected) "Other Gateways" else "Discovered Gateways", + style = MaterialTheme.typography.titleSmall, + ) + } + if (!isConnected && visibleGateways.isEmpty()) { + item { Text("No gateways found yet.", color = MaterialTheme.colorScheme.onSurfaceVariant) } + } else { + items(items = visibleGateways, key = { it.stableId }) { gateway -> + val detailLines = + buildList { + add("IP: ${gateway.host}:${gateway.port}") + gateway.lanHost?.let { add("LAN: $it") } + gateway.tailnetDns?.let { add("Tailnet: $it") } + if (gateway.gatewayPort != null || gateway.canvasPort != null) { + val gw = (gateway.gatewayPort ?: gateway.port).toString() + val canvas = gateway.canvasPort?.toString() ?: "—" + add("Ports: gw $gw · canvas $canvas") + } + } + ListItem( + headlineContent = { Text(gateway.name) }, + supportingContent = { + Column(verticalArrangement = Arrangement.spacedBy(2.dp)) { + detailLines.forEach { line -> + Text(line, color = MaterialTheme.colorScheme.onSurfaceVariant) + } + } + }, + trailingContent = { + Button( + onClick = { + NodeForegroundService.start(context) + viewModel.connect(gateway) + }, + ) { + Text("Connect") + } + }, + ) + } + } + item { + Text( + gatewayDiscoveryFooterText, + modifier = Modifier.fillMaxWidth(), + textAlign = TextAlign.Center, + style = MaterialTheme.typography.labelMedium, + color = MaterialTheme.colorScheme.onSurfaceVariant, + ) + } + } + + item { HorizontalDivider() } + + item { + ListItem( + headlineContent = { Text("Advanced") }, + supportingContent = { Text("Manual gateway connection") }, + trailingContent = { + Icon( + imageVector = if (advancedExpanded) Icons.Filled.ExpandLess else Icons.Filled.ExpandMore, + contentDescription = if (advancedExpanded) "Collapse" else "Expand", + ) + }, + modifier = + Modifier.clickable { + setAdvancedExpanded(!advancedExpanded) + }, + ) + } + item { + AnimatedVisibility(visible = advancedExpanded) { + Column(verticalArrangement = Arrangement.spacedBy(10.dp), modifier = Modifier.fillMaxWidth()) { + ListItem( + headlineContent = { Text("Use Manual Gateway") }, + supportingContent = { Text("Use this when discovery is blocked.") }, + trailingContent = { Switch(checked = manualEnabled, onCheckedChange = viewModel::setManualEnabled) }, + ) + + OutlinedTextField( + value = manualHost, + onValueChange = viewModel::setManualHost, + label = { Text("Host") }, + modifier = Modifier.fillMaxWidth(), + enabled = manualEnabled, + ) + OutlinedTextField( + value = manualPort.toString(), + onValueChange = { v -> viewModel.setManualPort(v.toIntOrNull() ?: 0) }, + label = { Text("Port") }, + modifier = Modifier.fillMaxWidth(), + enabled = manualEnabled, + ) + ListItem( + headlineContent = { Text("Require TLS") }, + supportingContent = { Text("Pin the gateway certificate on first connect.") }, + trailingContent = { Switch(checked = manualTls, onCheckedChange = viewModel::setManualTls, enabled = manualEnabled) }, + modifier = Modifier.alpha(if (manualEnabled) 1f else 0.5f), + ) + + val hostOk = manualHost.trim().isNotEmpty() + val portOk = manualPort in 1..65535 + Button( + onClick = { + NodeForegroundService.start(context) + viewModel.connectManual() + }, + enabled = manualEnabled && hostOk && portOk, + ) { + Text("Connect (Manual)") + } + } + } + } + + item { HorizontalDivider() } + + // Voice + item { Text("Voice", style = MaterialTheme.typography.titleSmall) } + item { + val enabled = voiceWakeMode != VoiceWakeMode.Off + ListItem( + headlineContent = { Text("Voice Wake") }, + supportingContent = { Text(voiceWakeStatusText) }, + trailingContent = { + Switch( + checked = enabled, + onCheckedChange = { on -> + if (on) { + val micOk = + ContextCompat.checkSelfPermission(context, Manifest.permission.RECORD_AUDIO) == + PackageManager.PERMISSION_GRANTED + if (!micOk) audioPermissionLauncher.launch(Manifest.permission.RECORD_AUDIO) + viewModel.setVoiceWakeMode(VoiceWakeMode.Foreground) + } else { + viewModel.setVoiceWakeMode(VoiceWakeMode.Off) + } + }, + ) + }, + ) + } + item { + AnimatedVisibility(visible = voiceWakeMode != VoiceWakeMode.Off) { + Column(verticalArrangement = Arrangement.spacedBy(6.dp), modifier = Modifier.fillMaxWidth()) { + ListItem( + headlineContent = { Text("Foreground Only") }, + supportingContent = { Text("Listens only while OpenClaw is open.") }, + trailingContent = { + RadioButton( + selected = voiceWakeMode == VoiceWakeMode.Foreground, + onClick = { + val micOk = + ContextCompat.checkSelfPermission(context, Manifest.permission.RECORD_AUDIO) == + PackageManager.PERMISSION_GRANTED + if (!micOk) audioPermissionLauncher.launch(Manifest.permission.RECORD_AUDIO) + viewModel.setVoiceWakeMode(VoiceWakeMode.Foreground) + }, + ) + }, + ) + ListItem( + headlineContent = { Text("Always") }, + supportingContent = { Text("Keeps listening in the background (shows a persistent notification).") }, + trailingContent = { + RadioButton( + selected = voiceWakeMode == VoiceWakeMode.Always, + onClick = { + val micOk = + ContextCompat.checkSelfPermission(context, Manifest.permission.RECORD_AUDIO) == + PackageManager.PERMISSION_GRANTED + if (!micOk) audioPermissionLauncher.launch(Manifest.permission.RECORD_AUDIO) + viewModel.setVoiceWakeMode(VoiceWakeMode.Always) + }, + ) + }, + ) + } + } + } + item { + OutlinedTextField( + value = wakeWordsText, + onValueChange = setWakeWordsText, + label = { Text("Wake Words (comma-separated)") }, + modifier = + Modifier.fillMaxWidth().onFocusChanged { focusState -> + if (focusState.isFocused) { + wakeWordsHadFocus = true + } else if (wakeWordsHadFocus) { + wakeWordsHadFocus = false + commitWakeWords() + } + }, + singleLine = true, + keyboardOptions = KeyboardOptions(imeAction = ImeAction.Done), + keyboardActions = + KeyboardActions( + onDone = { + commitWakeWords() + focusManager.clearFocus() + }, + ), + ) + } + item { Button(onClick = viewModel::resetWakeWordsDefaults) { Text("Reset defaults") } } + item { + Text( + if (isConnected) { + "Any node can edit wake words. Changes sync via the gateway." + } else { + "Connect to a gateway to sync wake words globally." + }, + color = MaterialTheme.colorScheme.onSurfaceVariant, + ) + } + + item { HorizontalDivider() } + + // Camera + item { Text("Camera", style = MaterialTheme.typography.titleSmall) } + item { + ListItem( + headlineContent = { Text("Allow Camera") }, + supportingContent = { Text("Allows the gateway to request photos or short video clips (foreground only).") }, + trailingContent = { Switch(checked = cameraEnabled, onCheckedChange = ::setCameraEnabledChecked) }, + ) + } + item { + Text( + "Tip: grant Microphone permission for video clips with audio.", + color = MaterialTheme.colorScheme.onSurfaceVariant, + ) + } + + item { HorizontalDivider() } + + // Messaging + item { Text("Messaging", style = MaterialTheme.typography.titleSmall) } + item { + val buttonLabel = + when { + !smsPermissionAvailable -> "Unavailable" + smsPermissionGranted -> "Manage" + else -> "Grant" + } + ListItem( + headlineContent = { Text("SMS Permission") }, + supportingContent = { + Text( + if (smsPermissionAvailable) { + "Allow the gateway to send SMS from this device." + } else { + "SMS requires a device with telephony hardware." + }, + ) + }, + trailingContent = { + Button( + onClick = { + if (!smsPermissionAvailable) return@Button + if (smsPermissionGranted) { + openAppSettings(context) + } else { + smsPermissionLauncher.launch(Manifest.permission.SEND_SMS) + } + }, + enabled = smsPermissionAvailable, + ) { + Text(buttonLabel) + } + }, + ) + } + + item { HorizontalDivider() } + + // Location + item { Text("Location", style = MaterialTheme.typography.titleSmall) } + item { + Column(verticalArrangement = Arrangement.spacedBy(6.dp), modifier = Modifier.fillMaxWidth()) { + ListItem( + headlineContent = { Text("Off") }, + supportingContent = { Text("Disable location sharing.") }, + trailingContent = { + RadioButton( + selected = locationMode == LocationMode.Off, + onClick = { viewModel.setLocationMode(LocationMode.Off) }, + ) + }, + ) + ListItem( + headlineContent = { Text("While Using") }, + supportingContent = { Text("Only while OpenClaw is open.") }, + trailingContent = { + RadioButton( + selected = locationMode == LocationMode.WhileUsing, + onClick = { requestLocationPermissions(LocationMode.WhileUsing) }, + ) + }, + ) + ListItem( + headlineContent = { Text("Always") }, + supportingContent = { Text("Allow background location (requires system permission).") }, + trailingContent = { + RadioButton( + selected = locationMode == LocationMode.Always, + onClick = { requestLocationPermissions(LocationMode.Always) }, + ) + }, + ) + } + } + item { + ListItem( + headlineContent = { Text("Precise Location") }, + supportingContent = { Text("Use precise GPS when available.") }, + trailingContent = { + Switch( + checked = locationPreciseEnabled, + onCheckedChange = ::setPreciseLocationChecked, + enabled = locationMode != LocationMode.Off, + ) + }, + ) + } + item { + Text( + "Always may require Android Settings to allow background location.", + color = MaterialTheme.colorScheme.onSurfaceVariant, + ) + } + + item { HorizontalDivider() } + + // Screen + item { Text("Screen", style = MaterialTheme.typography.titleSmall) } + item { + ListItem( + headlineContent = { Text("Prevent Sleep") }, + supportingContent = { Text("Keeps the screen awake while OpenClaw is open.") }, + trailingContent = { Switch(checked = preventSleep, onCheckedChange = viewModel::setPreventSleep) }, + ) + } + + item { HorizontalDivider() } + + // Debug + item { Text("Debug", style = MaterialTheme.typography.titleSmall) } + item { + ListItem( + headlineContent = { Text("Debug Canvas Status") }, + supportingContent = { Text("Show status text in the canvas when debug is enabled.") }, + trailingContent = { + Switch( + checked = canvasDebugStatusEnabled, + onCheckedChange = viewModel::setCanvasDebugStatusEnabled, + ) + }, + ) + } + + item { Spacer(modifier = Modifier.height(20.dp)) } + } +} + +private fun openAppSettings(context: Context) { + val intent = + Intent( + Settings.ACTION_APPLICATION_DETAILS_SETTINGS, + Uri.fromParts("package", context.packageName, null), + ) + context.startActivity(intent) +} diff --git a/apps/android/app/src/main/java/ai/openclaw/android/ui/StatusPill.kt b/apps/android/app/src/main/java/ai/openclaw/android/ui/StatusPill.kt new file mode 100644 index 0000000000000000000000000000000000000000..d608fc38a7bb48cdb63a5e71ec5a8ccb5c6e8866 --- /dev/null +++ b/apps/android/app/src/main/java/ai/openclaw/android/ui/StatusPill.kt @@ -0,0 +1,114 @@ +package ai.openclaw.android.ui + +import androidx.compose.foundation.layout.Arrangement +import androidx.compose.foundation.layout.Row +import androidx.compose.foundation.layout.Spacer +import androidx.compose.foundation.layout.height +import androidx.compose.foundation.layout.padding +import androidx.compose.foundation.layout.size +import androidx.compose.foundation.layout.width +import androidx.compose.foundation.shape.CircleShape +import androidx.compose.foundation.shape.RoundedCornerShape +import androidx.compose.material.icons.Icons +import androidx.compose.material.icons.filled.Mic +import androidx.compose.material.icons.filled.MicOff +import androidx.compose.material3.Icon +import androidx.compose.material3.MaterialTheme +import androidx.compose.material3.Surface +import androidx.compose.material3.Text +import androidx.compose.material3.VerticalDivider +import androidx.compose.runtime.Composable +import androidx.compose.ui.Alignment +import androidx.compose.ui.Modifier +import androidx.compose.ui.draw.alpha +import androidx.compose.ui.graphics.Color +import androidx.compose.ui.unit.dp + +@Composable +fun StatusPill( + gateway: GatewayState, + voiceEnabled: Boolean, + onClick: () -> Unit, + modifier: Modifier = Modifier, + activity: StatusActivity? = null, +) { + Surface( + onClick = onClick, + modifier = modifier, + shape = RoundedCornerShape(14.dp), + color = overlayContainerColor(), + tonalElevation = 3.dp, + shadowElevation = 0.dp, + ) { + Row( + modifier = Modifier.padding(horizontal = 12.dp, vertical = 8.dp), + horizontalArrangement = Arrangement.spacedBy(10.dp), + verticalAlignment = Alignment.CenterVertically, + ) { + Row(horizontalArrangement = Arrangement.spacedBy(8.dp), verticalAlignment = Alignment.CenterVertically) { + Surface( + modifier = Modifier.size(9.dp), + shape = CircleShape, + color = gateway.color, + ) {} + + Text( + text = gateway.title, + style = MaterialTheme.typography.labelLarge, + ) + } + + VerticalDivider( + modifier = Modifier.height(14.dp).alpha(0.35f), + color = MaterialTheme.colorScheme.onSurfaceVariant, + ) + + if (activity != null) { + Row( + horizontalArrangement = Arrangement.spacedBy(6.dp), + verticalAlignment = Alignment.CenterVertically, + ) { + Icon( + imageVector = activity.icon, + contentDescription = activity.contentDescription, + tint = activity.tint ?: overlayIconColor(), + modifier = Modifier.size(18.dp), + ) + Text( + text = activity.title, + style = MaterialTheme.typography.labelLarge, + maxLines = 1, + ) + } + } else { + Icon( + imageVector = if (voiceEnabled) Icons.Default.Mic else Icons.Default.MicOff, + contentDescription = if (voiceEnabled) "Voice enabled" else "Voice disabled", + tint = + if (voiceEnabled) { + overlayIconColor() + } else { + MaterialTheme.colorScheme.onSurfaceVariant + }, + modifier = Modifier.size(18.dp), + ) + } + + Spacer(modifier = Modifier.width(2.dp)) + } + } +} + +data class StatusActivity( + val title: String, + val icon: androidx.compose.ui.graphics.vector.ImageVector, + val contentDescription: String, + val tint: Color? = null, +) + +enum class GatewayState(val title: String, val color: Color) { + Connected("Connected", Color(0xFF2ECC71)), + Connecting("Connecting…", Color(0xFFF1C40F)), + Error("Error", Color(0xFFE74C3C)), + Disconnected("Offline", Color(0xFF9E9E9E)), +} diff --git a/apps/android/app/src/main/java/ai/openclaw/android/ui/TalkOrbOverlay.kt b/apps/android/app/src/main/java/ai/openclaw/android/ui/TalkOrbOverlay.kt new file mode 100644 index 0000000000000000000000000000000000000000..f89b298d1f772329b1dfed4993afc674ac487446 --- /dev/null +++ b/apps/android/app/src/main/java/ai/openclaw/android/ui/TalkOrbOverlay.kt @@ -0,0 +1,134 @@ +package ai.openclaw.android.ui + +import androidx.compose.animation.core.LinearEasing +import androidx.compose.animation.core.RepeatMode +import androidx.compose.animation.core.animateFloat +import androidx.compose.animation.core.infiniteRepeatable +import androidx.compose.animation.core.rememberInfiniteTransition +import androidx.compose.animation.core.tween +import androidx.compose.foundation.Canvas +import androidx.compose.foundation.layout.Arrangement +import androidx.compose.foundation.layout.Box +import androidx.compose.foundation.layout.Column +import androidx.compose.foundation.layout.padding +import androidx.compose.foundation.layout.size +import androidx.compose.foundation.shape.CircleShape +import androidx.compose.material3.MaterialTheme +import androidx.compose.material3.Surface +import androidx.compose.material3.Text +import androidx.compose.runtime.Composable +import androidx.compose.runtime.getValue +import androidx.compose.ui.Alignment +import androidx.compose.ui.Modifier +import androidx.compose.ui.graphics.Brush +import androidx.compose.ui.graphics.Color +import androidx.compose.ui.graphics.drawscope.Stroke +import androidx.compose.ui.text.font.FontWeight +import androidx.compose.ui.unit.dp + +@Composable +fun TalkOrbOverlay( + seamColor: Color, + statusText: String, + isListening: Boolean, + isSpeaking: Boolean, + modifier: Modifier = Modifier, +) { + val transition = rememberInfiniteTransition(label = "talk-orb") + val t by + transition.animateFloat( + initialValue = 0f, + targetValue = 1f, + animationSpec = + infiniteRepeatable( + animation = tween(durationMillis = 1500, easing = LinearEasing), + repeatMode = RepeatMode.Restart, + ), + label = "pulse", + ) + + val trimmed = statusText.trim() + val showStatus = trimmed.isNotEmpty() && trimmed != "Off" + val phase = + when { + isSpeaking -> "Speaking" + isListening -> "Listening" + else -> "Thinking" + } + + Column( + modifier = modifier.padding(24.dp), + horizontalAlignment = Alignment.CenterHorizontally, + verticalArrangement = Arrangement.spacedBy(12.dp), + ) { + Box(contentAlignment = Alignment.Center) { + Canvas(modifier = Modifier.size(360.dp)) { + val center = this.center + val baseRadius = size.minDimension * 0.30f + + val ring1 = 1.05f + (t * 0.25f) + val ring2 = 1.20f + (t * 0.55f) + val ringAlpha1 = (1f - t) * 0.34f + val ringAlpha2 = (1f - t) * 0.22f + + drawCircle( + color = seamColor.copy(alpha = ringAlpha1), + radius = baseRadius * ring1, + center = center, + style = Stroke(width = 3.dp.toPx()), + ) + drawCircle( + color = seamColor.copy(alpha = ringAlpha2), + radius = baseRadius * ring2, + center = center, + style = Stroke(width = 3.dp.toPx()), + ) + + drawCircle( + brush = + Brush.radialGradient( + colors = + listOf( + seamColor.copy(alpha = 0.92f), + seamColor.copy(alpha = 0.40f), + Color.Black.copy(alpha = 0.56f), + ), + center = center, + radius = baseRadius * 1.35f, + ), + radius = baseRadius, + center = center, + ) + + drawCircle( + color = seamColor.copy(alpha = 0.34f), + radius = baseRadius, + center = center, + style = Stroke(width = 1.dp.toPx()), + ) + } + } + + if (showStatus) { + Surface( + color = Color.Black.copy(alpha = 0.40f), + shape = CircleShape, + ) { + Text( + text = trimmed, + modifier = Modifier.padding(horizontal = 14.dp, vertical = 8.dp), + color = Color.White.copy(alpha = 0.92f), + style = MaterialTheme.typography.labelLarge, + fontWeight = FontWeight.SemiBold, + ) + } + } else { + Text( + text = phase, + color = Color.White.copy(alpha = 0.80f), + style = MaterialTheme.typography.labelLarge, + fontWeight = FontWeight.SemiBold, + ) + } + } +} diff --git a/apps/android/app/src/main/java/ai/openclaw/android/ui/chat/ChatComposer.kt b/apps/android/app/src/main/java/ai/openclaw/android/ui/chat/ChatComposer.kt new file mode 100644 index 0000000000000000000000000000000000000000..492516b51b17f4a494d476b9ae6ab7ed72ee2424 --- /dev/null +++ b/apps/android/app/src/main/java/ai/openclaw/android/ui/chat/ChatComposer.kt @@ -0,0 +1,285 @@ +package ai.openclaw.android.ui.chat + +import androidx.compose.foundation.layout.Arrangement +import androidx.compose.foundation.layout.Box +import androidx.compose.foundation.layout.Column +import androidx.compose.foundation.layout.Row +import androidx.compose.foundation.layout.Spacer +import androidx.compose.foundation.layout.fillMaxWidth +import androidx.compose.foundation.layout.padding +import androidx.compose.foundation.layout.size +import androidx.compose.foundation.layout.width +import androidx.compose.foundation.rememberScrollState +import androidx.compose.foundation.shape.RoundedCornerShape +import androidx.compose.foundation.horizontalScroll +import androidx.compose.material.icons.Icons +import androidx.compose.material.icons.filled.ArrowUpward +import androidx.compose.material.icons.filled.AttachFile +import androidx.compose.material.icons.filled.Refresh +import androidx.compose.material.icons.filled.Stop +import androidx.compose.material3.ButtonDefaults +import androidx.compose.material3.DropdownMenu +import androidx.compose.material3.DropdownMenuItem +import androidx.compose.material3.FilledTonalButton +import androidx.compose.material3.FilledTonalIconButton +import androidx.compose.material3.Icon +import androidx.compose.material3.IconButtonDefaults +import androidx.compose.material3.MaterialTheme +import androidx.compose.material3.OutlinedTextField +import androidx.compose.material3.Surface +import androidx.compose.material3.Text +import androidx.compose.runtime.Composable +import androidx.compose.runtime.getValue +import androidx.compose.runtime.mutableStateOf +import androidx.compose.runtime.remember +import androidx.compose.runtime.saveable.rememberSaveable +import androidx.compose.runtime.setValue +import androidx.compose.ui.Alignment +import androidx.compose.ui.Modifier +import androidx.compose.ui.graphics.Color +import androidx.compose.ui.unit.dp +import ai.openclaw.android.chat.ChatSessionEntry + +@Composable +fun ChatComposer( + sessionKey: String, + sessions: List, + mainSessionKey: String, + healthOk: Boolean, + thinkingLevel: String, + pendingRunCount: Int, + errorText: String?, + attachments: List, + onPickImages: () -> Unit, + onRemoveAttachment: (id: String) -> Unit, + onSetThinkingLevel: (level: String) -> Unit, + onSelectSession: (sessionKey: String) -> Unit, + onRefresh: () -> Unit, + onAbort: () -> Unit, + onSend: (text: String) -> Unit, +) { + var input by rememberSaveable { mutableStateOf("") } + var showThinkingMenu by remember { mutableStateOf(false) } + var showSessionMenu by remember { mutableStateOf(false) } + + val sessionOptions = resolveSessionChoices(sessionKey, sessions, mainSessionKey = mainSessionKey) + val currentSessionLabel = + sessionOptions.firstOrNull { it.key == sessionKey }?.displayName ?: sessionKey + + val canSend = pendingRunCount == 0 && (input.trim().isNotEmpty() || attachments.isNotEmpty()) && healthOk + + Surface( + shape = MaterialTheme.shapes.large, + color = MaterialTheme.colorScheme.surfaceContainer, + tonalElevation = 0.dp, + shadowElevation = 0.dp, + ) { + Column(modifier = Modifier.padding(10.dp), verticalArrangement = Arrangement.spacedBy(8.dp)) { + Row( + modifier = Modifier.fillMaxWidth(), + horizontalArrangement = Arrangement.spacedBy(8.dp), + verticalAlignment = Alignment.CenterVertically, + ) { + Box { + FilledTonalButton( + onClick = { showSessionMenu = true }, + contentPadding = ButtonDefaults.ContentPadding, + ) { + Text("Session: $currentSessionLabel") + } + + DropdownMenu(expanded = showSessionMenu, onDismissRequest = { showSessionMenu = false }) { + for (entry in sessionOptions) { + DropdownMenuItem( + text = { Text(entry.displayName ?: entry.key) }, + onClick = { + onSelectSession(entry.key) + showSessionMenu = false + }, + trailingIcon = { + if (entry.key == sessionKey) { + Text("✓") + } else { + Spacer(modifier = Modifier.width(10.dp)) + } + }, + ) + } + } + } + + Box { + FilledTonalButton( + onClick = { showThinkingMenu = true }, + contentPadding = ButtonDefaults.ContentPadding, + ) { + Text("Thinking: ${thinkingLabel(thinkingLevel)}") + } + + DropdownMenu(expanded = showThinkingMenu, onDismissRequest = { showThinkingMenu = false }) { + ThinkingMenuItem("off", thinkingLevel, onSetThinkingLevel) { showThinkingMenu = false } + ThinkingMenuItem("low", thinkingLevel, onSetThinkingLevel) { showThinkingMenu = false } + ThinkingMenuItem("medium", thinkingLevel, onSetThinkingLevel) { showThinkingMenu = false } + ThinkingMenuItem("high", thinkingLevel, onSetThinkingLevel) { showThinkingMenu = false } + } + } + + Spacer(modifier = Modifier.weight(1f)) + + FilledTonalIconButton(onClick = onRefresh, modifier = Modifier.size(42.dp)) { + Icon(Icons.Default.Refresh, contentDescription = "Refresh") + } + + FilledTonalIconButton(onClick = onPickImages, modifier = Modifier.size(42.dp)) { + Icon(Icons.Default.AttachFile, contentDescription = "Add image") + } + } + + if (attachments.isNotEmpty()) { + AttachmentsStrip(attachments = attachments, onRemoveAttachment = onRemoveAttachment) + } + + OutlinedTextField( + value = input, + onValueChange = { input = it }, + modifier = Modifier.fillMaxWidth(), + placeholder = { Text("Message OpenClaw…") }, + minLines = 2, + maxLines = 6, + ) + + Row(modifier = Modifier.fillMaxWidth(), verticalAlignment = Alignment.CenterVertically) { + ConnectionPill(sessionLabel = currentSessionLabel, healthOk = healthOk) + Spacer(modifier = Modifier.weight(1f)) + + if (pendingRunCount > 0) { + FilledTonalIconButton( + onClick = onAbort, + colors = + IconButtonDefaults.filledTonalIconButtonColors( + containerColor = Color(0x33E74C3C), + contentColor = Color(0xFFE74C3C), + ), + ) { + Icon(Icons.Default.Stop, contentDescription = "Abort") + } + } else { + FilledTonalIconButton(onClick = { + val text = input + input = "" + onSend(text) + }, enabled = canSend) { + Icon(Icons.Default.ArrowUpward, contentDescription = "Send") + } + } + } + + if (!errorText.isNullOrBlank()) { + Text( + text = errorText, + style = MaterialTheme.typography.bodySmall, + color = MaterialTheme.colorScheme.error, + maxLines = 2, + ) + } + } + } +} + +@Composable +private fun ConnectionPill(sessionLabel: String, healthOk: Boolean) { + Surface( + shape = RoundedCornerShape(999.dp), + color = MaterialTheme.colorScheme.surfaceContainerHighest, + ) { + Row( + modifier = Modifier.padding(horizontal = 10.dp, vertical = 6.dp), + horizontalArrangement = Arrangement.spacedBy(8.dp), + verticalAlignment = Alignment.CenterVertically, + ) { + Surface( + modifier = Modifier.size(7.dp), + shape = androidx.compose.foundation.shape.CircleShape, + color = if (healthOk) Color(0xFF2ECC71) else Color(0xFFF39C12), + ) {} + Text(sessionLabel, style = MaterialTheme.typography.labelSmall) + Text( + if (healthOk) "Connected" else "Connecting…", + style = MaterialTheme.typography.labelSmall, + color = MaterialTheme.colorScheme.onSurfaceVariant, + ) + } + } +} + +@Composable +private fun ThinkingMenuItem( + value: String, + current: String, + onSet: (String) -> Unit, + onDismiss: () -> Unit, +) { + DropdownMenuItem( + text = { Text(thinkingLabel(value)) }, + onClick = { + onSet(value) + onDismiss() + }, + trailingIcon = { + if (value == current.trim().lowercase()) { + Text("✓") + } else { + Spacer(modifier = Modifier.width(10.dp)) + } + }, + ) +} + +private fun thinkingLabel(raw: String): String { + return when (raw.trim().lowercase()) { + "low" -> "Low" + "medium" -> "Medium" + "high" -> "High" + else -> "Off" + } +} + +@Composable +private fun AttachmentsStrip( + attachments: List, + onRemoveAttachment: (id: String) -> Unit, +) { + Row( + modifier = Modifier.fillMaxWidth().horizontalScroll(rememberScrollState()), + horizontalArrangement = Arrangement.spacedBy(8.dp), + ) { + for (att in attachments) { + AttachmentChip( + fileName = att.fileName, + onRemove = { onRemoveAttachment(att.id) }, + ) + } + } +} + +@Composable +private fun AttachmentChip(fileName: String, onRemove: () -> Unit) { + Surface( + shape = RoundedCornerShape(999.dp), + color = MaterialTheme.colorScheme.primary.copy(alpha = 0.10f), + ) { + Row( + modifier = Modifier.padding(horizontal = 10.dp, vertical = 6.dp), + verticalAlignment = Alignment.CenterVertically, + horizontalArrangement = Arrangement.spacedBy(8.dp), + ) { + Text(text = fileName, style = MaterialTheme.typography.bodySmall, maxLines = 1) + FilledTonalIconButton( + onClick = onRemove, + modifier = Modifier.size(30.dp), + ) { + Text("×") + } + } + } +} diff --git a/apps/android/app/src/main/java/ai/openclaw/android/ui/chat/ChatMarkdown.kt b/apps/android/app/src/main/java/ai/openclaw/android/ui/chat/ChatMarkdown.kt new file mode 100644 index 0000000000000000000000000000000000000000..77dba2275a41e7150861aacd46b3a8c4515b3328 --- /dev/null +++ b/apps/android/app/src/main/java/ai/openclaw/android/ui/chat/ChatMarkdown.kt @@ -0,0 +1,215 @@ +package ai.openclaw.android.ui.chat + +import android.graphics.BitmapFactory +import android.util.Base64 +import androidx.compose.foundation.Image +import androidx.compose.foundation.layout.Arrangement +import androidx.compose.foundation.layout.Column +import androidx.compose.foundation.layout.fillMaxWidth +import androidx.compose.foundation.layout.padding +import androidx.compose.foundation.text.selection.SelectionContainer +import androidx.compose.material3.MaterialTheme +import androidx.compose.material3.Text +import androidx.compose.runtime.Composable +import androidx.compose.runtime.LaunchedEffect +import androidx.compose.runtime.getValue +import androidx.compose.runtime.mutableStateOf +import androidx.compose.runtime.remember +import androidx.compose.runtime.setValue +import androidx.compose.ui.Modifier +import androidx.compose.ui.graphics.Color +import androidx.compose.ui.graphics.asImageBitmap +import androidx.compose.ui.layout.ContentScale +import androidx.compose.ui.text.AnnotatedString +import androidx.compose.ui.text.SpanStyle +import androidx.compose.ui.text.buildAnnotatedString +import androidx.compose.ui.text.font.FontFamily +import androidx.compose.ui.text.font.FontStyle +import androidx.compose.ui.text.font.FontWeight +import androidx.compose.ui.text.withStyle +import androidx.compose.ui.unit.dp +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext + +@Composable +fun ChatMarkdown(text: String, textColor: Color) { + val blocks = remember(text) { splitMarkdown(text) } + val inlineCodeBg = MaterialTheme.colorScheme.surfaceContainerLow + + Column(verticalArrangement = Arrangement.spacedBy(10.dp)) { + for (b in blocks) { + when (b) { + is ChatMarkdownBlock.Text -> { + val trimmed = b.text.trimEnd() + if (trimmed.isEmpty()) continue + Text( + text = parseInlineMarkdown(trimmed, inlineCodeBg = inlineCodeBg), + style = MaterialTheme.typography.bodyMedium, + color = textColor, + ) + } + is ChatMarkdownBlock.Code -> { + SelectionContainer(modifier = Modifier.fillMaxWidth()) { + ChatCodeBlock(code = b.code, language = b.language) + } + } + is ChatMarkdownBlock.InlineImage -> { + InlineBase64Image(base64 = b.base64, mimeType = b.mimeType) + } + } + } + } +} + +private sealed interface ChatMarkdownBlock { + data class Text(val text: String) : ChatMarkdownBlock + data class Code(val code: String, val language: String?) : ChatMarkdownBlock + data class InlineImage(val mimeType: String?, val base64: String) : ChatMarkdownBlock +} + +private fun splitMarkdown(raw: String): List { + if (raw.isEmpty()) return emptyList() + + val out = ArrayList() + var idx = 0 + while (idx < raw.length) { + val fenceStart = raw.indexOf("```", startIndex = idx) + if (fenceStart < 0) { + out.addAll(splitInlineImages(raw.substring(idx))) + break + } + + if (fenceStart > idx) { + out.addAll(splitInlineImages(raw.substring(idx, fenceStart))) + } + + val langLineStart = fenceStart + 3 + val langLineEnd = raw.indexOf('\n', startIndex = langLineStart).let { if (it < 0) raw.length else it } + val language = raw.substring(langLineStart, langLineEnd).trim().ifEmpty { null } + + val codeStart = if (langLineEnd < raw.length && raw[langLineEnd] == '\n') langLineEnd + 1 else langLineEnd + val fenceEnd = raw.indexOf("```", startIndex = codeStart) + if (fenceEnd < 0) { + out.addAll(splitInlineImages(raw.substring(fenceStart))) + break + } + val code = raw.substring(codeStart, fenceEnd) + out.add(ChatMarkdownBlock.Code(code = code, language = language)) + + idx = fenceEnd + 3 + } + + return out +} + +private fun splitInlineImages(text: String): List { + if (text.isEmpty()) return emptyList() + val regex = Regex("data:image/([a-zA-Z0-9+.-]+);base64,([A-Za-z0-9+/=\\n\\r]+)") + val out = ArrayList() + + var idx = 0 + while (idx < text.length) { + val m = regex.find(text, startIndex = idx) ?: break + val start = m.range.first + val end = m.range.last + 1 + if (start > idx) out.add(ChatMarkdownBlock.Text(text.substring(idx, start))) + + val mime = "image/" + (m.groupValues.getOrNull(1)?.trim()?.ifEmpty { "png" } ?: "png") + val b64 = m.groupValues.getOrNull(2)?.replace("\n", "")?.replace("\r", "")?.trim().orEmpty() + if (b64.isNotEmpty()) { + out.add(ChatMarkdownBlock.InlineImage(mimeType = mime, base64 = b64)) + } + idx = end + } + + if (idx < text.length) out.add(ChatMarkdownBlock.Text(text.substring(idx))) + return out +} + +private fun parseInlineMarkdown(text: String, inlineCodeBg: androidx.compose.ui.graphics.Color): AnnotatedString { + if (text.isEmpty()) return AnnotatedString("") + + val out = buildAnnotatedString { + var i = 0 + while (i < text.length) { + if (text.startsWith("**", startIndex = i)) { + val end = text.indexOf("**", startIndex = i + 2) + if (end > i + 2) { + withStyle(SpanStyle(fontWeight = FontWeight.SemiBold)) { + append(text.substring(i + 2, end)) + } + i = end + 2 + continue + } + } + + if (text[i] == '`') { + val end = text.indexOf('`', startIndex = i + 1) + if (end > i + 1) { + withStyle( + SpanStyle( + fontFamily = FontFamily.Monospace, + background = inlineCodeBg, + ), + ) { + append(text.substring(i + 1, end)) + } + i = end + 1 + continue + } + } + + if (text[i] == '*' && (i + 1 < text.length && text[i + 1] != '*')) { + val end = text.indexOf('*', startIndex = i + 1) + if (end > i + 1) { + withStyle(SpanStyle(fontStyle = FontStyle.Italic)) { + append(text.substring(i + 1, end)) + } + i = end + 1 + continue + } + } + + append(text[i]) + i += 1 + } + } + return out +} + +@Composable +private fun InlineBase64Image(base64: String, mimeType: String?) { + var image by remember(base64) { mutableStateOf(null) } + var failed by remember(base64) { mutableStateOf(false) } + + LaunchedEffect(base64) { + failed = false + image = + withContext(Dispatchers.Default) { + try { + val bytes = Base64.decode(base64, Base64.DEFAULT) + val bitmap = BitmapFactory.decodeByteArray(bytes, 0, bytes.size) ?: return@withContext null + bitmap.asImageBitmap() + } catch (_: Throwable) { + null + } + } + if (image == null) failed = true + } + + if (image != null) { + Image( + bitmap = image!!, + contentDescription = mimeType ?: "image", + contentScale = ContentScale.Fit, + modifier = Modifier.fillMaxWidth(), + ) + } else if (failed) { + Text( + text = "Image unavailable", + modifier = Modifier.padding(vertical = 2.dp), + style = MaterialTheme.typography.bodySmall, + color = MaterialTheme.colorScheme.onSurfaceVariant, + ) + } +} diff --git a/apps/android/app/src/main/java/ai/openclaw/android/ui/chat/ChatMessageListCard.kt b/apps/android/app/src/main/java/ai/openclaw/android/ui/chat/ChatMessageListCard.kt new file mode 100644 index 0000000000000000000000000000000000000000..d2634637297c4649cdb22079f00f4bac47165f07 --- /dev/null +++ b/apps/android/app/src/main/java/ai/openclaw/android/ui/chat/ChatMessageListCard.kt @@ -0,0 +1,111 @@ +package ai.openclaw.android.ui.chat + +import androidx.compose.foundation.layout.Arrangement +import androidx.compose.foundation.layout.Box +import androidx.compose.foundation.layout.Row +import androidx.compose.foundation.layout.fillMaxSize +import androidx.compose.foundation.layout.fillMaxWidth +import androidx.compose.foundation.lazy.LazyColumn +import androidx.compose.foundation.lazy.rememberLazyListState +import androidx.compose.material.icons.Icons +import androidx.compose.material.icons.filled.ArrowCircleDown +import androidx.compose.material3.Card +import androidx.compose.material3.CardDefaults +import androidx.compose.material3.Icon +import androidx.compose.material3.MaterialTheme +import androidx.compose.material3.Text +import androidx.compose.runtime.Composable +import androidx.compose.runtime.LaunchedEffect +import androidx.compose.ui.Alignment +import androidx.compose.ui.Modifier +import androidx.compose.ui.draw.alpha +import androidx.compose.ui.unit.dp +import ai.openclaw.android.chat.ChatMessage +import ai.openclaw.android.chat.ChatPendingToolCall + +@Composable +fun ChatMessageListCard( + messages: List, + pendingRunCount: Int, + pendingToolCalls: List, + streamingAssistantText: String?, + modifier: Modifier = Modifier, +) { + val listState = rememberLazyListState() + + LaunchedEffect(messages.size, pendingRunCount, pendingToolCalls.size, streamingAssistantText) { + val total = + messages.size + + (if (pendingRunCount > 0) 1 else 0) + + (if (pendingToolCalls.isNotEmpty()) 1 else 0) + + (if (!streamingAssistantText.isNullOrBlank()) 1 else 0) + if (total <= 0) return@LaunchedEffect + listState.animateScrollToItem(index = total - 1) + } + + Card( + modifier = modifier.fillMaxWidth(), + shape = MaterialTheme.shapes.large, + colors = + CardDefaults.cardColors( + containerColor = MaterialTheme.colorScheme.surfaceContainer, + ), + elevation = CardDefaults.cardElevation(defaultElevation = 0.dp), + ) { + Box(modifier = Modifier.fillMaxSize()) { + LazyColumn( + modifier = Modifier.fillMaxSize(), + state = listState, + verticalArrangement = Arrangement.spacedBy(14.dp), + contentPadding = androidx.compose.foundation.layout.PaddingValues(top = 12.dp, bottom = 12.dp, start = 12.dp, end = 12.dp), + ) { + items(count = messages.size, key = { idx -> messages[idx].id }) { idx -> + ChatMessageBubble(message = messages[idx]) + } + + if (pendingRunCount > 0) { + item(key = "typing") { + ChatTypingIndicatorBubble() + } + } + + if (pendingToolCalls.isNotEmpty()) { + item(key = "tools") { + ChatPendingToolsBubble(toolCalls = pendingToolCalls) + } + } + + val stream = streamingAssistantText?.trim() + if (!stream.isNullOrEmpty()) { + item(key = "stream") { + ChatStreamingAssistantBubble(text = stream) + } + } + } + + if (messages.isEmpty() && pendingRunCount == 0 && pendingToolCalls.isEmpty() && streamingAssistantText.isNullOrBlank()) { + EmptyChatHint(modifier = Modifier.align(Alignment.Center)) + } + } + } +} + +@Composable +private fun EmptyChatHint(modifier: Modifier = Modifier) { + Row( + modifier = modifier.alpha(0.7f), + verticalAlignment = Alignment.CenterVertically, + horizontalArrangement = Arrangement.spacedBy(8.dp), + ) { + Icon( + imageVector = Icons.Default.ArrowCircleDown, + contentDescription = null, + tint = MaterialTheme.colorScheme.onSurfaceVariant, + ) + Text( + text = "Message OpenClaw…", + style = MaterialTheme.typography.bodyMedium, + color = MaterialTheme.colorScheme.onSurfaceVariant, + ) + } +} diff --git a/apps/android/app/src/main/java/ai/openclaw/android/ui/chat/ChatMessageViews.kt b/apps/android/app/src/main/java/ai/openclaw/android/ui/chat/ChatMessageViews.kt new file mode 100644 index 0000000000000000000000000000000000000000..1f87db32a54c9576273fa07e662f430bc18fa8a9 --- /dev/null +++ b/apps/android/app/src/main/java/ai/openclaw/android/ui/chat/ChatMessageViews.kt @@ -0,0 +1,252 @@ +package ai.openclaw.android.ui.chat + +import android.graphics.BitmapFactory +import android.util.Base64 +import androidx.compose.foundation.background +import androidx.compose.foundation.layout.Arrangement +import androidx.compose.foundation.layout.Box +import androidx.compose.foundation.layout.Column +import androidx.compose.foundation.layout.Row +import androidx.compose.foundation.layout.fillMaxWidth +import androidx.compose.foundation.layout.padding +import androidx.compose.foundation.layout.size +import androidx.compose.foundation.shape.CircleShape +import androidx.compose.foundation.shape.RoundedCornerShape +import androidx.compose.material3.MaterialTheme +import androidx.compose.material3.Surface +import androidx.compose.material3.Text +import androidx.compose.runtime.Composable +import androidx.compose.runtime.LaunchedEffect +import androidx.compose.runtime.getValue +import androidx.compose.runtime.mutableStateOf +import androidx.compose.runtime.remember +import androidx.compose.runtime.setValue +import androidx.compose.ui.Alignment +import androidx.compose.ui.Modifier +import androidx.compose.ui.draw.alpha +import androidx.compose.ui.graphics.Brush +import androidx.compose.ui.graphics.Color +import androidx.compose.ui.graphics.asImageBitmap +import androidx.compose.ui.layout.ContentScale +import androidx.compose.ui.text.font.FontFamily +import androidx.compose.ui.unit.dp +import androidx.compose.foundation.Image +import ai.openclaw.android.chat.ChatMessage +import ai.openclaw.android.chat.ChatMessageContent +import ai.openclaw.android.chat.ChatPendingToolCall +import ai.openclaw.android.tools.ToolDisplayRegistry +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext +import androidx.compose.ui.platform.LocalContext + +@Composable +fun ChatMessageBubble(message: ChatMessage) { + val isUser = message.role.lowercase() == "user" + + Row( + modifier = Modifier.fillMaxWidth(), + horizontalArrangement = if (isUser) Arrangement.End else Arrangement.Start, + ) { + Surface( + shape = RoundedCornerShape(16.dp), + tonalElevation = 0.dp, + shadowElevation = 0.dp, + color = Color.Transparent, + modifier = Modifier.fillMaxWidth(0.92f), + ) { + Box( + modifier = + Modifier + .background(bubbleBackground(isUser)) + .padding(horizontal = 12.dp, vertical = 10.dp), + ) { + val textColor = textColorOverBubble(isUser) + ChatMessageBody(content = message.content, textColor = textColor) + } + } + } +} + +@Composable +private fun ChatMessageBody(content: List, textColor: Color) { + Column(verticalArrangement = Arrangement.spacedBy(10.dp)) { + for (part in content) { + when (part.type) { + "text" -> { + val text = part.text ?: continue + ChatMarkdown(text = text, textColor = textColor) + } + else -> { + val b64 = part.base64 ?: continue + ChatBase64Image(base64 = b64, mimeType = part.mimeType) + } + } + } + } +} + +@Composable +fun ChatTypingIndicatorBubble() { + Row(modifier = Modifier.fillMaxWidth(), horizontalArrangement = Arrangement.Start) { + Surface( + shape = RoundedCornerShape(16.dp), + color = MaterialTheme.colorScheme.surfaceContainer, + ) { + Row( + modifier = Modifier.padding(horizontal = 12.dp, vertical = 10.dp), + verticalAlignment = Alignment.CenterVertically, + horizontalArrangement = Arrangement.spacedBy(8.dp), + ) { + DotPulse() + Text("Thinking…", style = MaterialTheme.typography.bodyMedium, color = MaterialTheme.colorScheme.onSurfaceVariant) + } + } + } +} + +@Composable +fun ChatPendingToolsBubble(toolCalls: List) { + val context = LocalContext.current + val displays = + remember(toolCalls, context) { + toolCalls.map { ToolDisplayRegistry.resolve(context, it.name, it.args) } + } + Row(modifier = Modifier.fillMaxWidth(), horizontalArrangement = Arrangement.Start) { + Surface( + shape = RoundedCornerShape(16.dp), + color = MaterialTheme.colorScheme.surfaceContainer, + ) { + Column(modifier = Modifier.padding(horizontal = 12.dp, vertical = 10.dp), verticalArrangement = Arrangement.spacedBy(6.dp)) { + Text("Running tools…", style = MaterialTheme.typography.labelLarge, color = MaterialTheme.colorScheme.onSurface) + for (display in displays.take(6)) { + Column(verticalArrangement = Arrangement.spacedBy(2.dp)) { + Text( + "${display.emoji} ${display.label}", + style = MaterialTheme.typography.bodyMedium, + color = MaterialTheme.colorScheme.onSurfaceVariant, + fontFamily = FontFamily.Monospace, + ) + display.detailLine?.let { detail -> + Text( + detail, + style = MaterialTheme.typography.bodySmall, + color = MaterialTheme.colorScheme.onSurfaceVariant, + fontFamily = FontFamily.Monospace, + ) + } + } + } + if (toolCalls.size > 6) { + Text( + "… +${toolCalls.size - 6} more", + style = MaterialTheme.typography.bodySmall, + color = MaterialTheme.colorScheme.onSurfaceVariant, + ) + } + } + } + } +} + +@Composable +fun ChatStreamingAssistantBubble(text: String) { + Row(modifier = Modifier.fillMaxWidth(), horizontalArrangement = Arrangement.Start) { + Surface( + shape = RoundedCornerShape(16.dp), + color = MaterialTheme.colorScheme.surfaceContainer, + ) { + Box(modifier = Modifier.padding(horizontal = 12.dp, vertical = 10.dp)) { + ChatMarkdown(text = text, textColor = MaterialTheme.colorScheme.onSurface) + } + } + } +} + +@Composable +private fun bubbleBackground(isUser: Boolean): Brush { + return if (isUser) { + Brush.linearGradient( + colors = listOf(MaterialTheme.colorScheme.primary, MaterialTheme.colorScheme.primary.copy(alpha = 0.78f)), + ) + } else { + Brush.linearGradient( + colors = listOf(MaterialTheme.colorScheme.surfaceContainer, MaterialTheme.colorScheme.surfaceContainerHigh), + ) + } +} + +@Composable +private fun textColorOverBubble(isUser: Boolean): Color { + return if (isUser) { + MaterialTheme.colorScheme.onPrimary + } else { + MaterialTheme.colorScheme.onSurface + } +} + +@Composable +private fun ChatBase64Image(base64: String, mimeType: String?) { + var image by remember(base64) { mutableStateOf(null) } + var failed by remember(base64) { mutableStateOf(false) } + + LaunchedEffect(base64) { + failed = false + image = + withContext(Dispatchers.Default) { + try { + val bytes = Base64.decode(base64, Base64.DEFAULT) + val bitmap = BitmapFactory.decodeByteArray(bytes, 0, bytes.size) ?: return@withContext null + bitmap.asImageBitmap() + } catch (_: Throwable) { + null + } + } + if (image == null) failed = true + } + + if (image != null) { + Image( + bitmap = image!!, + contentDescription = mimeType ?: "attachment", + contentScale = ContentScale.Fit, + modifier = Modifier.fillMaxWidth(), + ) + } else if (failed) { + Text("Unsupported attachment", style = MaterialTheme.typography.bodySmall, color = MaterialTheme.colorScheme.onSurfaceVariant) + } +} + +@Composable +private fun DotPulse() { + Row(horizontalArrangement = Arrangement.spacedBy(5.dp), verticalAlignment = Alignment.CenterVertically) { + PulseDot(alpha = 0.38f) + PulseDot(alpha = 0.62f) + PulseDot(alpha = 0.90f) + } +} + +@Composable +private fun PulseDot(alpha: Float) { + Surface( + modifier = Modifier.size(6.dp).alpha(alpha), + shape = CircleShape, + color = MaterialTheme.colorScheme.onSurfaceVariant, + ) {} +} + +@Composable +fun ChatCodeBlock(code: String, language: String?) { + Surface( + shape = RoundedCornerShape(12.dp), + color = MaterialTheme.colorScheme.surfaceContainerLowest, + modifier = Modifier.fillMaxWidth(), + ) { + Text( + text = code.trimEnd(), + modifier = Modifier.padding(10.dp), + fontFamily = FontFamily.Monospace, + style = MaterialTheme.typography.bodySmall, + color = MaterialTheme.colorScheme.onSurface, + ) + } +} diff --git a/apps/android/app/src/main/java/ai/openclaw/android/ui/chat/ChatSessionsDialog.kt b/apps/android/app/src/main/java/ai/openclaw/android/ui/chat/ChatSessionsDialog.kt new file mode 100644 index 0000000000000000000000000000000000000000..56b5cfb1faf67120a6d050d34498ddf036a0a91e --- /dev/null +++ b/apps/android/app/src/main/java/ai/openclaw/android/ui/chat/ChatSessionsDialog.kt @@ -0,0 +1,92 @@ +package ai.openclaw.android.ui.chat + +import androidx.compose.foundation.layout.Arrangement +import androidx.compose.foundation.layout.Column +import androidx.compose.foundation.layout.Row +import androidx.compose.foundation.layout.Spacer +import androidx.compose.foundation.layout.fillMaxWidth +import androidx.compose.foundation.layout.padding +import androidx.compose.foundation.lazy.LazyColumn +import androidx.compose.foundation.lazy.items +import androidx.compose.material.icons.Icons +import androidx.compose.material.icons.filled.Refresh +import androidx.compose.material3.AlertDialog +import androidx.compose.material3.FilledTonalIconButton +import androidx.compose.material3.Icon +import androidx.compose.material3.MaterialTheme +import androidx.compose.material3.Surface +import androidx.compose.material3.Text +import androidx.compose.runtime.Composable +import androidx.compose.ui.Alignment +import androidx.compose.ui.Modifier +import androidx.compose.ui.unit.dp +import ai.openclaw.android.chat.ChatSessionEntry + +@Composable +fun ChatSessionsDialog( + currentSessionKey: String, + sessions: List, + onDismiss: () -> Unit, + onRefresh: () -> Unit, + onSelect: (sessionKey: String) -> Unit, +) { + AlertDialog( + onDismissRequest = onDismiss, + confirmButton = {}, + title = { + Row(verticalAlignment = Alignment.CenterVertically, modifier = Modifier.fillMaxWidth()) { + Text("Sessions", style = MaterialTheme.typography.titleMedium) + Spacer(modifier = Modifier.weight(1f)) + FilledTonalIconButton(onClick = onRefresh) { + Icon(Icons.Default.Refresh, contentDescription = "Refresh") + } + } + }, + text = { + if (sessions.isEmpty()) { + Text("No sessions", style = MaterialTheme.typography.bodyMedium, color = MaterialTheme.colorScheme.onSurfaceVariant) + } else { + LazyColumn(verticalArrangement = Arrangement.spacedBy(8.dp)) { + items(sessions, key = { it.key }) { entry -> + SessionRow( + entry = entry, + isCurrent = entry.key == currentSessionKey, + onClick = { onSelect(entry.key) }, + ) + } + } + } + }, + ) +} + +@Composable +private fun SessionRow( + entry: ChatSessionEntry, + isCurrent: Boolean, + onClick: () -> Unit, +) { + Surface( + onClick = onClick, + shape = MaterialTheme.shapes.medium, + color = + if (isCurrent) { + MaterialTheme.colorScheme.primary.copy(alpha = 0.14f) + } else { + MaterialTheme.colorScheme.surfaceContainer + }, + modifier = Modifier.fillMaxWidth(), + ) { + Row( + modifier = Modifier.padding(horizontal = 12.dp, vertical = 10.dp), + verticalAlignment = Alignment.CenterVertically, + horizontalArrangement = Arrangement.spacedBy(10.dp), + ) { + Text(entry.displayName ?: entry.key, style = MaterialTheme.typography.bodyMedium) + Spacer(modifier = Modifier.weight(1f)) + if (isCurrent) { + Text("Current", style = MaterialTheme.typography.labelSmall, color = MaterialTheme.colorScheme.onSurfaceVariant) + } + } + } +} diff --git a/apps/android/app/src/main/java/ai/openclaw/android/ui/chat/ChatSheetContent.kt b/apps/android/app/src/main/java/ai/openclaw/android/ui/chat/ChatSheetContent.kt new file mode 100644 index 0000000000000000000000000000000000000000..effee6708e0d10a9652a8010b729281a8d20c64f --- /dev/null +++ b/apps/android/app/src/main/java/ai/openclaw/android/ui/chat/ChatSheetContent.kt @@ -0,0 +1,147 @@ +package ai.openclaw.android.ui.chat + +import android.content.ContentResolver +import android.net.Uri +import android.util.Base64 +import androidx.activity.compose.rememberLauncherForActivityResult +import androidx.activity.result.contract.ActivityResultContracts +import androidx.compose.foundation.layout.Arrangement +import androidx.compose.foundation.layout.Column +import androidx.compose.foundation.layout.fillMaxSize +import androidx.compose.foundation.layout.padding +import androidx.compose.runtime.Composable +import androidx.compose.runtime.LaunchedEffect +import androidx.compose.runtime.collectAsState +import androidx.compose.runtime.getValue +import androidx.compose.runtime.mutableStateListOf +import androidx.compose.runtime.remember +import androidx.compose.runtime.rememberCoroutineScope +import androidx.compose.ui.Modifier +import androidx.compose.ui.platform.LocalContext +import androidx.compose.ui.unit.dp +import ai.openclaw.android.MainViewModel +import ai.openclaw.android.chat.OutgoingAttachment +import java.io.ByteArrayOutputStream +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.launch +import kotlinx.coroutines.withContext + +@Composable +fun ChatSheetContent(viewModel: MainViewModel) { + val messages by viewModel.chatMessages.collectAsState() + val errorText by viewModel.chatError.collectAsState() + val pendingRunCount by viewModel.pendingRunCount.collectAsState() + val healthOk by viewModel.chatHealthOk.collectAsState() + val sessionKey by viewModel.chatSessionKey.collectAsState() + val mainSessionKey by viewModel.mainSessionKey.collectAsState() + val thinkingLevel by viewModel.chatThinkingLevel.collectAsState() + val streamingAssistantText by viewModel.chatStreamingAssistantText.collectAsState() + val pendingToolCalls by viewModel.chatPendingToolCalls.collectAsState() + val sessions by viewModel.chatSessions.collectAsState() + + LaunchedEffect(mainSessionKey) { + viewModel.loadChat(mainSessionKey) + viewModel.refreshChatSessions(limit = 200) + } + + val context = LocalContext.current + val resolver = context.contentResolver + val scope = rememberCoroutineScope() + + val attachments = remember { mutableStateListOf() } + + val pickImages = + rememberLauncherForActivityResult(ActivityResultContracts.GetMultipleContents()) { uris -> + if (uris.isNullOrEmpty()) return@rememberLauncherForActivityResult + scope.launch(Dispatchers.IO) { + val next = + uris.take(8).mapNotNull { uri -> + try { + loadImageAttachment(resolver, uri) + } catch (_: Throwable) { + null + } + } + withContext(Dispatchers.Main) { + attachments.addAll(next) + } + } + } + + Column( + modifier = + Modifier + .fillMaxSize() + .padding(horizontal = 12.dp, vertical = 12.dp), + verticalArrangement = Arrangement.spacedBy(10.dp), + ) { + ChatMessageListCard( + messages = messages, + pendingRunCount = pendingRunCount, + pendingToolCalls = pendingToolCalls, + streamingAssistantText = streamingAssistantText, + modifier = Modifier.weight(1f, fill = true), + ) + + ChatComposer( + sessionKey = sessionKey, + sessions = sessions, + mainSessionKey = mainSessionKey, + healthOk = healthOk, + thinkingLevel = thinkingLevel, + pendingRunCount = pendingRunCount, + errorText = errorText, + attachments = attachments, + onPickImages = { pickImages.launch("image/*") }, + onRemoveAttachment = { id -> attachments.removeAll { it.id == id } }, + onSetThinkingLevel = { level -> viewModel.setChatThinkingLevel(level) }, + onSelectSession = { key -> viewModel.switchChatSession(key) }, + onRefresh = { + viewModel.refreshChat() + viewModel.refreshChatSessions(limit = 200) + }, + onAbort = { viewModel.abortChat() }, + onSend = { text -> + val outgoing = + attachments.map { att -> + OutgoingAttachment( + type = "image", + mimeType = att.mimeType, + fileName = att.fileName, + base64 = att.base64, + ) + } + viewModel.sendChat(message = text, thinking = thinkingLevel, attachments = outgoing) + attachments.clear() + }, + ) + } +} + +data class PendingImageAttachment( + val id: String, + val fileName: String, + val mimeType: String, + val base64: String, +) + +private suspend fun loadImageAttachment(resolver: ContentResolver, uri: Uri): PendingImageAttachment { + val mimeType = resolver.getType(uri) ?: "image/*" + val fileName = (uri.lastPathSegment ?: "image").substringAfterLast('/') + val bytes = + withContext(Dispatchers.IO) { + resolver.openInputStream(uri)?.use { input -> + val out = ByteArrayOutputStream() + input.copyTo(out) + out.toByteArray() + } ?: ByteArray(0) + } + if (bytes.isEmpty()) throw IllegalStateException("empty attachment") + val base64 = Base64.encodeToString(bytes, Base64.NO_WRAP) + return PendingImageAttachment( + id = uri.toString() + "#" + System.currentTimeMillis().toString(), + fileName = fileName, + mimeType = mimeType, + base64 = base64, + ) +} diff --git a/apps/android/app/src/main/java/ai/openclaw/android/ui/chat/SessionFilters.kt b/apps/android/app/src/main/java/ai/openclaw/android/ui/chat/SessionFilters.kt new file mode 100644 index 0000000000000000000000000000000000000000..4efca2d0cf30a603150eb92748b046aedfb640db --- /dev/null +++ b/apps/android/app/src/main/java/ai/openclaw/android/ui/chat/SessionFilters.kt @@ -0,0 +1,49 @@ +package ai.openclaw.android.ui.chat + +import ai.openclaw.android.chat.ChatSessionEntry + +private const val RECENT_WINDOW_MS = 24 * 60 * 60 * 1000L + +fun resolveSessionChoices( + currentSessionKey: String, + sessions: List, + mainSessionKey: String, + nowMs: Long = System.currentTimeMillis(), +): List { + val mainKey = mainSessionKey.trim().ifEmpty { "main" } + val current = currentSessionKey.trim().let { if (it == "main" && mainKey != "main") mainKey else it } + val aliasKey = if (mainKey == "main") null else "main" + val cutoff = nowMs - RECENT_WINDOW_MS + val sorted = sessions.sortedByDescending { it.updatedAtMs ?: 0L } + val recent = mutableListOf() + val seen = mutableSetOf() + for (entry in sorted) { + if (aliasKey != null && entry.key == aliasKey) continue + if (!seen.add(entry.key)) continue + if ((entry.updatedAtMs ?: 0L) < cutoff) continue + recent.add(entry) + } + + val result = mutableListOf() + val included = mutableSetOf() + val mainEntry = sorted.firstOrNull { it.key == mainKey } + if (mainEntry != null) { + result.add(mainEntry) + included.add(mainKey) + } else if (current == mainKey) { + result.add(ChatSessionEntry(key = mainKey, updatedAtMs = null)) + included.add(mainKey) + } + + for (entry in recent) { + if (included.add(entry.key)) { + result.add(entry) + } + } + + if (current.isNotEmpty() && !included.contains(current)) { + result.add(ChatSessionEntry(key = current, updatedAtMs = null)) + } + + return result +} diff --git a/apps/android/app/src/main/java/ai/openclaw/android/voice/StreamingMediaDataSource.kt b/apps/android/app/src/main/java/ai/openclaw/android/voice/StreamingMediaDataSource.kt new file mode 100644 index 0000000000000000000000000000000000000000..329707ad56ac1892a467e9ac4c493f038b282cf7 --- /dev/null +++ b/apps/android/app/src/main/java/ai/openclaw/android/voice/StreamingMediaDataSource.kt @@ -0,0 +1,98 @@ +package ai.openclaw.android.voice + +import android.media.MediaDataSource +import kotlin.math.min + +internal class StreamingMediaDataSource : MediaDataSource() { + private data class Chunk(val start: Long, val data: ByteArray) + + private val lock = Object() + private val chunks = ArrayList() + private var totalSize: Long = 0 + private var closed = false + private var finished = false + private var lastReadIndex = 0 + + fun append(data: ByteArray) { + if (data.isEmpty()) return + synchronized(lock) { + if (closed || finished) return + val chunk = Chunk(totalSize, data) + chunks.add(chunk) + totalSize += data.size.toLong() + lock.notifyAll() + } + } + + fun finish() { + synchronized(lock) { + if (closed) return + finished = true + lock.notifyAll() + } + } + + fun fail() { + synchronized(lock) { + closed = true + lock.notifyAll() + } + } + + override fun readAt(position: Long, buffer: ByteArray, offset: Int, size: Int): Int { + if (position < 0) return -1 + synchronized(lock) { + while (!closed && !finished && position >= totalSize) { + lock.wait() + } + if (closed) return -1 + if (position >= totalSize && finished) return -1 + + val available = (totalSize - position).toInt() + val toRead = min(size, available) + var remaining = toRead + var destOffset = offset + var pos = position + + var index = findChunkIndex(pos) + while (remaining > 0 && index < chunks.size) { + val chunk = chunks[index] + val inChunkOffset = (pos - chunk.start).toInt() + if (inChunkOffset >= chunk.data.size) { + index++ + continue + } + val copyLen = min(remaining, chunk.data.size - inChunkOffset) + System.arraycopy(chunk.data, inChunkOffset, buffer, destOffset, copyLen) + remaining -= copyLen + destOffset += copyLen + pos += copyLen + if (inChunkOffset + copyLen >= chunk.data.size) { + index++ + } + } + + return toRead - remaining + } + } + + override fun getSize(): Long = -1 + + override fun close() { + synchronized(lock) { + closed = true + lock.notifyAll() + } + } + + private fun findChunkIndex(position: Long): Int { + var index = lastReadIndex + while (index < chunks.size) { + val chunk = chunks[index] + if (position < chunk.start + chunk.data.size) break + index++ + } + lastReadIndex = index + return index + } +} diff --git a/apps/android/app/src/main/java/ai/openclaw/android/voice/TalkDirectiveParser.kt b/apps/android/app/src/main/java/ai/openclaw/android/voice/TalkDirectiveParser.kt new file mode 100644 index 0000000000000000000000000000000000000000..5c80cc1f4f1bfbd7069101e72d54c68d7066f713 --- /dev/null +++ b/apps/android/app/src/main/java/ai/openclaw/android/voice/TalkDirectiveParser.kt @@ -0,0 +1,191 @@ +package ai.openclaw.android.voice + +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.JsonElement +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.JsonPrimitive + +private val directiveJson = Json { ignoreUnknownKeys = true } + +data class TalkDirective( + val voiceId: String? = null, + val modelId: String? = null, + val speed: Double? = null, + val rateWpm: Int? = null, + val stability: Double? = null, + val similarity: Double? = null, + val style: Double? = null, + val speakerBoost: Boolean? = null, + val seed: Long? = null, + val normalize: String? = null, + val language: String? = null, + val outputFormat: String? = null, + val latencyTier: Int? = null, + val once: Boolean? = null, +) + +data class TalkDirectiveParseResult( + val directive: TalkDirective?, + val stripped: String, + val unknownKeys: List, +) + +object TalkDirectiveParser { + fun parse(text: String): TalkDirectiveParseResult { + val normalized = text.replace("\r\n", "\n") + val lines = normalized.split("\n").toMutableList() + if (lines.isEmpty()) return TalkDirectiveParseResult(null, text, emptyList()) + + val firstNonEmpty = lines.indexOfFirst { it.trim().isNotEmpty() } + if (firstNonEmpty == -1) return TalkDirectiveParseResult(null, text, emptyList()) + + val head = lines[firstNonEmpty].trim() + if (!head.startsWith("{") || !head.endsWith("}")) { + return TalkDirectiveParseResult(null, text, emptyList()) + } + + val obj = parseJsonObject(head) ?: return TalkDirectiveParseResult(null, text, emptyList()) + + val speakerBoost = + boolValue(obj, listOf("speaker_boost", "speakerBoost")) + ?: boolValue(obj, listOf("no_speaker_boost", "noSpeakerBoost"))?.not() + + val directive = TalkDirective( + voiceId = stringValue(obj, listOf("voice", "voice_id", "voiceId")), + modelId = stringValue(obj, listOf("model", "model_id", "modelId")), + speed = doubleValue(obj, listOf("speed")), + rateWpm = intValue(obj, listOf("rate", "wpm")), + stability = doubleValue(obj, listOf("stability")), + similarity = doubleValue(obj, listOf("similarity", "similarity_boost", "similarityBoost")), + style = doubleValue(obj, listOf("style")), + speakerBoost = speakerBoost, + seed = longValue(obj, listOf("seed")), + normalize = stringValue(obj, listOf("normalize", "apply_text_normalization")), + language = stringValue(obj, listOf("lang", "language_code", "language")), + outputFormat = stringValue(obj, listOf("output_format", "format")), + latencyTier = intValue(obj, listOf("latency", "latency_tier", "latencyTier")), + once = boolValue(obj, listOf("once")), + ) + + val hasDirective = listOf( + directive.voiceId, + directive.modelId, + directive.speed, + directive.rateWpm, + directive.stability, + directive.similarity, + directive.style, + directive.speakerBoost, + directive.seed, + directive.normalize, + directive.language, + directive.outputFormat, + directive.latencyTier, + directive.once, + ).any { it != null } + + if (!hasDirective) return TalkDirectiveParseResult(null, text, emptyList()) + + val knownKeys = setOf( + "voice", "voice_id", "voiceid", + "model", "model_id", "modelid", + "speed", "rate", "wpm", + "stability", "similarity", "similarity_boost", "similarityboost", + "style", + "speaker_boost", "speakerboost", + "no_speaker_boost", "nospeakerboost", + "seed", + "normalize", "apply_text_normalization", + "lang", "language_code", "language", + "output_format", "format", + "latency", "latency_tier", "latencytier", + "once", + ) + val unknownKeys = obj.keys.filter { !knownKeys.contains(it.lowercase()) }.sorted() + + lines.removeAt(firstNonEmpty) + if (firstNonEmpty < lines.size) { + if (lines[firstNonEmpty].trim().isEmpty()) { + lines.removeAt(firstNonEmpty) + } + } + + return TalkDirectiveParseResult(directive, lines.joinToString("\n"), unknownKeys) + } + + private fun parseJsonObject(line: String): JsonObject? { + return try { + directiveJson.parseToJsonElement(line) as? JsonObject + } catch (_: Throwable) { + null + } + } + + private fun stringValue(obj: JsonObject, keys: List): String? { + for (key in keys) { + val value = obj[key].asStringOrNull()?.trim() + if (!value.isNullOrEmpty()) return value + } + return null + } + + private fun doubleValue(obj: JsonObject, keys: List): Double? { + for (key in keys) { + val value = obj[key].asDoubleOrNull() + if (value != null) return value + } + return null + } + + private fun intValue(obj: JsonObject, keys: List): Int? { + for (key in keys) { + val value = obj[key].asIntOrNull() + if (value != null) return value + } + return null + } + + private fun longValue(obj: JsonObject, keys: List): Long? { + for (key in keys) { + val value = obj[key].asLongOrNull() + if (value != null) return value + } + return null + } + + private fun boolValue(obj: JsonObject, keys: List): Boolean? { + for (key in keys) { + val value = obj[key].asBooleanOrNull() + if (value != null) return value + } + return null + } +} + +private fun JsonElement?.asStringOrNull(): String? = + (this as? JsonPrimitive)?.takeIf { it.isString }?.content + +private fun JsonElement?.asDoubleOrNull(): Double? { + val primitive = this as? JsonPrimitive ?: return null + return primitive.content.toDoubleOrNull() +} + +private fun JsonElement?.asIntOrNull(): Int? { + val primitive = this as? JsonPrimitive ?: return null + return primitive.content.toIntOrNull() +} + +private fun JsonElement?.asLongOrNull(): Long? { + val primitive = this as? JsonPrimitive ?: return null + return primitive.content.toLongOrNull() +} + +private fun JsonElement?.asBooleanOrNull(): Boolean? { + val primitive = this as? JsonPrimitive ?: return null + val content = primitive.content.trim().lowercase() + return when (content) { + "true", "yes", "1" -> true + "false", "no", "0" -> false + else -> null + } +} diff --git a/apps/android/app/src/main/java/ai/openclaw/android/voice/TalkModeManager.kt b/apps/android/app/src/main/java/ai/openclaw/android/voice/TalkModeManager.kt new file mode 100644 index 0000000000000000000000000000000000000000..d4ca06f50fa8e410cbae39ae38d7c06acd8fe76e --- /dev/null +++ b/apps/android/app/src/main/java/ai/openclaw/android/voice/TalkModeManager.kt @@ -0,0 +1,1257 @@ +package ai.openclaw.android.voice + +import android.Manifest +import android.content.Context +import android.content.Intent +import android.content.pm.PackageManager +import android.media.AudioAttributes +import android.media.AudioFormat +import android.media.AudioManager +import android.media.AudioTrack +import android.media.MediaPlayer +import android.os.Bundle +import android.os.Handler +import android.os.Looper +import android.os.SystemClock +import android.speech.RecognitionListener +import android.speech.RecognizerIntent +import android.speech.SpeechRecognizer +import android.speech.tts.TextToSpeech +import android.speech.tts.UtteranceProgressListener +import android.util.Log +import androidx.core.content.ContextCompat +import ai.openclaw.android.gateway.GatewaySession +import ai.openclaw.android.isCanonicalMainSessionKey +import ai.openclaw.android.normalizeMainKey +import java.net.HttpURLConnection +import java.net.URL +import java.util.UUID +import kotlinx.coroutines.CompletableDeferred +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.Job +import kotlinx.coroutines.delay +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.launch +import kotlinx.coroutines.withContext +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.JsonArray +import kotlinx.serialization.json.JsonElement +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.JsonPrimitive +import kotlinx.serialization.json.buildJsonObject +import kotlin.math.max + +class TalkModeManager( + private val context: Context, + private val scope: CoroutineScope, + private val session: GatewaySession, + private val supportsChatSubscribe: Boolean, + private val isConnected: () -> Boolean, +) { + companion object { + private const val tag = "TalkMode" + private const val defaultModelIdFallback = "eleven_v3" + private const val defaultOutputFormatFallback = "pcm_24000" + } + + private val mainHandler = Handler(Looper.getMainLooper()) + private val json = Json { ignoreUnknownKeys = true } + + private val _isEnabled = MutableStateFlow(false) + val isEnabled: StateFlow = _isEnabled + + private val _isListening = MutableStateFlow(false) + val isListening: StateFlow = _isListening + + private val _isSpeaking = MutableStateFlow(false) + val isSpeaking: StateFlow = _isSpeaking + + private val _statusText = MutableStateFlow("Off") + val statusText: StateFlow = _statusText + + private val _lastAssistantText = MutableStateFlow(null) + val lastAssistantText: StateFlow = _lastAssistantText + + private val _usingFallbackTts = MutableStateFlow(false) + val usingFallbackTts: StateFlow = _usingFallbackTts + + private var recognizer: SpeechRecognizer? = null + private var restartJob: Job? = null + private var stopRequested = false + private var listeningMode = false + + private var silenceJob: Job? = null + private val silenceWindowMs = 700L + private var lastTranscript: String = "" + private var lastHeardAtMs: Long? = null + private var lastSpokenText: String? = null + private var lastInterruptedAtSeconds: Double? = null + + private var defaultVoiceId: String? = null + private var currentVoiceId: String? = null + private var fallbackVoiceId: String? = null + private var defaultModelId: String? = null + private var currentModelId: String? = null + private var defaultOutputFormat: String? = null + private var apiKey: String? = null + private var voiceAliases: Map = emptyMap() + private var interruptOnSpeech: Boolean = true + private var voiceOverrideActive = false + private var modelOverrideActive = false + private var mainSessionKey: String = "main" + + private var pendingRunId: String? = null + private var pendingFinal: CompletableDeferred? = null + private var chatSubscribedSessionKey: String? = null + + private var player: MediaPlayer? = null + private var streamingSource: StreamingMediaDataSource? = null + private var pcmTrack: AudioTrack? = null + @Volatile private var pcmStopRequested = false + private var systemTts: TextToSpeech? = null + private var systemTtsPending: CompletableDeferred? = null + private var systemTtsPendingId: String? = null + + fun setMainSessionKey(sessionKey: String?) { + val trimmed = sessionKey?.trim().orEmpty() + if (trimmed.isEmpty()) return + if (isCanonicalMainSessionKey(mainSessionKey)) return + mainSessionKey = trimmed + } + + fun setEnabled(enabled: Boolean) { + if (_isEnabled.value == enabled) return + _isEnabled.value = enabled + if (enabled) { + Log.d(tag, "enabled") + start() + } else { + Log.d(tag, "disabled") + stop() + } + } + + fun handleGatewayEvent(event: String, payloadJson: String?) { + if (event != "chat") return + if (payloadJson.isNullOrBlank()) return + val pending = pendingRunId ?: return + val obj = + try { + json.parseToJsonElement(payloadJson).asObjectOrNull() + } catch (_: Throwable) { + null + } ?: return + val runId = obj["runId"].asStringOrNull() ?: return + if (runId != pending) return + val state = obj["state"].asStringOrNull() ?: return + if (state == "final") { + pendingFinal?.complete(true) + pendingFinal = null + pendingRunId = null + } + } + + private fun start() { + mainHandler.post { + if (_isListening.value) return@post + stopRequested = false + listeningMode = true + Log.d(tag, "start") + + if (!SpeechRecognizer.isRecognitionAvailable(context)) { + _statusText.value = "Speech recognizer unavailable" + Log.w(tag, "speech recognizer unavailable") + return@post + } + + val micOk = + ContextCompat.checkSelfPermission(context, Manifest.permission.RECORD_AUDIO) == + PackageManager.PERMISSION_GRANTED + if (!micOk) { + _statusText.value = "Microphone permission required" + Log.w(tag, "microphone permission required") + return@post + } + + try { + recognizer?.destroy() + recognizer = SpeechRecognizer.createSpeechRecognizer(context).also { it.setRecognitionListener(listener) } + startListeningInternal(markListening = true) + startSilenceMonitor() + Log.d(tag, "listening") + } catch (err: Throwable) { + _statusText.value = "Start failed: ${err.message ?: err::class.simpleName}" + Log.w(tag, "start failed: ${err.message ?: err::class.simpleName}") + } + } + } + + private fun stop() { + stopRequested = true + listeningMode = false + restartJob?.cancel() + restartJob = null + silenceJob?.cancel() + silenceJob = null + lastTranscript = "" + lastHeardAtMs = null + _isListening.value = false + _statusText.value = "Off" + stopSpeaking() + _usingFallbackTts.value = false + chatSubscribedSessionKey = null + + mainHandler.post { + recognizer?.cancel() + recognizer?.destroy() + recognizer = null + } + systemTts?.stop() + systemTtsPending?.cancel() + systemTtsPending = null + systemTtsPendingId = null + } + + private fun startListeningInternal(markListening: Boolean) { + val r = recognizer ?: return + val intent = + Intent(RecognizerIntent.ACTION_RECOGNIZE_SPEECH).apply { + putExtra(RecognizerIntent.EXTRA_LANGUAGE_MODEL, RecognizerIntent.LANGUAGE_MODEL_FREE_FORM) + putExtra(RecognizerIntent.EXTRA_PARTIAL_RESULTS, true) + putExtra(RecognizerIntent.EXTRA_MAX_RESULTS, 3) + putExtra(RecognizerIntent.EXTRA_CALLING_PACKAGE, context.packageName) + } + + if (markListening) { + _statusText.value = "Listening" + _isListening.value = true + } + r.startListening(intent) + } + + private fun scheduleRestart(delayMs: Long = 350) { + if (stopRequested) return + restartJob?.cancel() + restartJob = + scope.launch { + delay(delayMs) + mainHandler.post { + if (stopRequested) return@post + try { + recognizer?.cancel() + val shouldListen = listeningMode + val shouldInterrupt = _isSpeaking.value && interruptOnSpeech + if (!shouldListen && !shouldInterrupt) return@post + startListeningInternal(markListening = shouldListen) + } catch (_: Throwable) { + // handled by onError + } + } + } + } + + private fun handleTranscript(text: String, isFinal: Boolean) { + val trimmed = text.trim() + if (_isSpeaking.value && interruptOnSpeech) { + if (shouldInterrupt(trimmed)) { + stopSpeaking() + } + return + } + + if (!_isListening.value) return + + if (trimmed.isNotEmpty()) { + lastTranscript = trimmed + lastHeardAtMs = SystemClock.elapsedRealtime() + } + + if (isFinal) { + lastTranscript = trimmed + } + } + + private fun startSilenceMonitor() { + silenceJob?.cancel() + silenceJob = + scope.launch { + while (_isEnabled.value) { + delay(200) + checkSilence() + } + } + } + + private fun checkSilence() { + if (!_isListening.value) return + val transcript = lastTranscript.trim() + if (transcript.isEmpty()) return + val lastHeard = lastHeardAtMs ?: return + val elapsed = SystemClock.elapsedRealtime() - lastHeard + if (elapsed < silenceWindowMs) return + scope.launch { finalizeTranscript(transcript) } + } + + private suspend fun finalizeTranscript(transcript: String) { + listeningMode = false + _isListening.value = false + _statusText.value = "Thinking…" + lastTranscript = "" + lastHeardAtMs = null + + reloadConfig() + val prompt = buildPrompt(transcript) + if (!isConnected()) { + _statusText.value = "Gateway not connected" + Log.w(tag, "finalize: gateway not connected") + start() + return + } + + try { + val startedAt = System.currentTimeMillis().toDouble() / 1000.0 + subscribeChatIfNeeded(session = session, sessionKey = mainSessionKey) + Log.d(tag, "chat.send start sessionKey=${mainSessionKey.ifBlank { "main" }} chars=${prompt.length}") + val runId = sendChat(prompt, session) + Log.d(tag, "chat.send ok runId=$runId") + val ok = waitForChatFinal(runId) + if (!ok) { + Log.w(tag, "chat final timeout runId=$runId; attempting history fallback") + } + val assistant = waitForAssistantText(session, startedAt, if (ok) 12_000 else 25_000) + if (assistant.isNullOrBlank()) { + _statusText.value = "No reply" + Log.w(tag, "assistant text timeout runId=$runId") + start() + return + } + Log.d(tag, "assistant text ok chars=${assistant.length}") + playAssistant(assistant) + } catch (err: Throwable) { + _statusText.value = "Talk failed: ${err.message ?: err::class.simpleName}" + Log.w(tag, "finalize failed: ${err.message ?: err::class.simpleName}") + } + + if (_isEnabled.value) { + start() + } + } + + private suspend fun subscribeChatIfNeeded(session: GatewaySession, sessionKey: String) { + if (!supportsChatSubscribe) return + val key = sessionKey.trim() + if (key.isEmpty()) return + if (chatSubscribedSessionKey == key) return + try { + session.sendNodeEvent("chat.subscribe", """{"sessionKey":"$key"}""") + chatSubscribedSessionKey = key + Log.d(tag, "chat.subscribe ok sessionKey=$key") + } catch (err: Throwable) { + Log.w(tag, "chat.subscribe failed sessionKey=$key err=${err.message ?: err::class.java.simpleName}") + } + } + + private fun buildPrompt(transcript: String): String { + val lines = mutableListOf( + "Talk Mode active. Reply in a concise, spoken tone.", + "You may optionally prefix the response with JSON (first line) to set ElevenLabs voice (id or alias), e.g. {\"voice\":\"\",\"once\":true}.", + ) + lastInterruptedAtSeconds?.let { + lines.add("Assistant speech interrupted at ${"%.1f".format(it)}s.") + lastInterruptedAtSeconds = null + } + lines.add("") + lines.add(transcript) + return lines.joinToString("\n") + } + + private suspend fun sendChat(message: String, session: GatewaySession): String { + val runId = UUID.randomUUID().toString() + val params = + buildJsonObject { + put("sessionKey", JsonPrimitive(mainSessionKey.ifBlank { "main" })) + put("message", JsonPrimitive(message)) + put("thinking", JsonPrimitive("low")) + put("timeoutMs", JsonPrimitive(30_000)) + put("idempotencyKey", JsonPrimitive(runId)) + } + val res = session.request("chat.send", params.toString()) + val parsed = parseRunId(res) ?: runId + if (parsed != runId) { + pendingRunId = parsed + } + return parsed + } + + private suspend fun waitForChatFinal(runId: String): Boolean { + pendingFinal?.cancel() + val deferred = CompletableDeferred() + pendingRunId = runId + pendingFinal = deferred + + val result = + withContext(Dispatchers.IO) { + try { + kotlinx.coroutines.withTimeout(120_000) { deferred.await() } + } catch (_: Throwable) { + false + } + } + + if (!result) { + pendingFinal = null + pendingRunId = null + } + return result + } + + private suspend fun waitForAssistantText( + session: GatewaySession, + sinceSeconds: Double, + timeoutMs: Long, + ): String? { + val deadline = SystemClock.elapsedRealtime() + timeoutMs + while (SystemClock.elapsedRealtime() < deadline) { + val text = fetchLatestAssistantText(session, sinceSeconds) + if (!text.isNullOrBlank()) return text + delay(300) + } + return null + } + + private suspend fun fetchLatestAssistantText( + session: GatewaySession, + sinceSeconds: Double? = null, + ): String? { + val key = mainSessionKey.ifBlank { "main" } + val res = session.request("chat.history", "{\"sessionKey\":\"$key\"}") + val root = json.parseToJsonElement(res).asObjectOrNull() ?: return null + val messages = root["messages"] as? JsonArray ?: return null + for (item in messages.reversed()) { + val obj = item.asObjectOrNull() ?: continue + if (obj["role"].asStringOrNull() != "assistant") continue + if (sinceSeconds != null) { + val timestamp = obj["timestamp"].asDoubleOrNull() + if (timestamp != null && !TalkModeRuntime.isMessageTimestampAfter(timestamp, sinceSeconds)) continue + } + val content = obj["content"] as? JsonArray ?: continue + val text = + content.mapNotNull { entry -> + entry.asObjectOrNull()?.get("text")?.asStringOrNull()?.trim() + }.filter { it.isNotEmpty() } + if (text.isNotEmpty()) return text.joinToString("\n") + } + return null + } + + private suspend fun playAssistant(text: String) { + val parsed = TalkDirectiveParser.parse(text) + if (parsed.unknownKeys.isNotEmpty()) { + Log.w(tag, "Unknown talk directive keys: ${parsed.unknownKeys}") + } + val directive = parsed.directive + val cleaned = parsed.stripped.trim() + if (cleaned.isEmpty()) return + _lastAssistantText.value = cleaned + + val requestedVoice = directive?.voiceId?.trim()?.takeIf { it.isNotEmpty() } + val resolvedVoice = resolveVoiceAlias(requestedVoice) + if (requestedVoice != null && resolvedVoice == null) { + Log.w(tag, "unknown voice alias: $requestedVoice") + } + + if (directive?.voiceId != null) { + if (directive.once != true) { + currentVoiceId = resolvedVoice + voiceOverrideActive = true + } + } + if (directive?.modelId != null) { + if (directive.once != true) { + currentModelId = directive.modelId + modelOverrideActive = true + } + } + + val apiKey = + apiKey?.trim()?.takeIf { it.isNotEmpty() } + ?: System.getenv("ELEVENLABS_API_KEY")?.trim() + val preferredVoice = resolvedVoice ?: currentVoiceId ?: defaultVoiceId + val voiceId = + if (!apiKey.isNullOrEmpty()) { + resolveVoiceId(preferredVoice, apiKey) + } else { + null + } + + _statusText.value = "Speaking…" + _isSpeaking.value = true + lastSpokenText = cleaned + ensureInterruptListener() + + try { + val canUseElevenLabs = !voiceId.isNullOrBlank() && !apiKey.isNullOrEmpty() + if (!canUseElevenLabs) { + if (voiceId.isNullOrBlank()) { + Log.w(tag, "missing voiceId; falling back to system voice") + } + if (apiKey.isNullOrEmpty()) { + Log.w(tag, "missing ELEVENLABS_API_KEY; falling back to system voice") + } + _usingFallbackTts.value = true + _statusText.value = "Speaking (System)…" + speakWithSystemTts(cleaned) + } else { + _usingFallbackTts.value = false + val ttsStarted = SystemClock.elapsedRealtime() + val modelId = directive?.modelId ?: currentModelId ?: defaultModelId + val request = + ElevenLabsRequest( + text = cleaned, + modelId = modelId, + outputFormat = + TalkModeRuntime.validatedOutputFormat(directive?.outputFormat ?: defaultOutputFormat), + speed = TalkModeRuntime.resolveSpeed(directive?.speed, directive?.rateWpm), + stability = TalkModeRuntime.validatedStability(directive?.stability, modelId), + similarity = TalkModeRuntime.validatedUnit(directive?.similarity), + style = TalkModeRuntime.validatedUnit(directive?.style), + speakerBoost = directive?.speakerBoost, + seed = TalkModeRuntime.validatedSeed(directive?.seed), + normalize = TalkModeRuntime.validatedNormalize(directive?.normalize), + language = TalkModeRuntime.validatedLanguage(directive?.language), + latencyTier = TalkModeRuntime.validatedLatencyTier(directive?.latencyTier), + ) + streamAndPlay(voiceId = voiceId!!, apiKey = apiKey!!, request = request) + Log.d(tag, "elevenlabs stream ok durMs=${SystemClock.elapsedRealtime() - ttsStarted}") + } + } catch (err: Throwable) { + Log.w(tag, "speak failed: ${err.message ?: err::class.simpleName}; falling back to system voice") + try { + _usingFallbackTts.value = true + _statusText.value = "Speaking (System)…" + speakWithSystemTts(cleaned) + } catch (fallbackErr: Throwable) { + _statusText.value = "Speak failed: ${fallbackErr.message ?: fallbackErr::class.simpleName}" + Log.w(tag, "system voice failed: ${fallbackErr.message ?: fallbackErr::class.simpleName}") + } + } + + _isSpeaking.value = false + } + + private suspend fun streamAndPlay(voiceId: String, apiKey: String, request: ElevenLabsRequest) { + stopSpeaking(resetInterrupt = false) + + pcmStopRequested = false + val pcmSampleRate = TalkModeRuntime.parsePcmSampleRate(request.outputFormat) + if (pcmSampleRate != null) { + try { + streamAndPlayPcm(voiceId = voiceId, apiKey = apiKey, request = request, sampleRate = pcmSampleRate) + return + } catch (err: Throwable) { + if (pcmStopRequested) return + Log.w(tag, "pcm playback failed; falling back to mp3: ${err.message ?: err::class.simpleName}") + } + } + + streamAndPlayMp3(voiceId = voiceId, apiKey = apiKey, request = request) + } + + private suspend fun streamAndPlayMp3(voiceId: String, apiKey: String, request: ElevenLabsRequest) { + val dataSource = StreamingMediaDataSource() + streamingSource = dataSource + + val player = MediaPlayer() + this.player = player + + val prepared = CompletableDeferred() + val finished = CompletableDeferred() + + player.setAudioAttributes( + AudioAttributes.Builder() + .setContentType(AudioAttributes.CONTENT_TYPE_SPEECH) + .setUsage(AudioAttributes.USAGE_ASSISTANT) + .build(), + ) + player.setOnPreparedListener { + it.start() + prepared.complete(Unit) + } + player.setOnCompletionListener { + finished.complete(Unit) + } + player.setOnErrorListener { _, _, _ -> + finished.completeExceptionally(IllegalStateException("MediaPlayer error")) + true + } + + player.setDataSource(dataSource) + withContext(Dispatchers.Main) { + player.prepareAsync() + } + + val fetchError = CompletableDeferred() + val fetchJob = + scope.launch(Dispatchers.IO) { + try { + streamTts(voiceId = voiceId, apiKey = apiKey, request = request, sink = dataSource) + fetchError.complete(null) + } catch (err: Throwable) { + dataSource.fail() + fetchError.complete(err) + } + } + + Log.d(tag, "play start") + try { + prepared.await() + finished.await() + fetchError.await()?.let { throw it } + } finally { + fetchJob.cancel() + cleanupPlayer() + } + Log.d(tag, "play done") + } + + private suspend fun streamAndPlayPcm( + voiceId: String, + apiKey: String, + request: ElevenLabsRequest, + sampleRate: Int, + ) { + val minBuffer = + AudioTrack.getMinBufferSize( + sampleRate, + AudioFormat.CHANNEL_OUT_MONO, + AudioFormat.ENCODING_PCM_16BIT, + ) + if (minBuffer <= 0) { + throw IllegalStateException("AudioTrack buffer size invalid: $minBuffer") + } + + val bufferSize = max(minBuffer * 2, 8 * 1024) + val track = + AudioTrack( + AudioAttributes.Builder() + .setContentType(AudioAttributes.CONTENT_TYPE_SPEECH) + .setUsage(AudioAttributes.USAGE_ASSISTANT) + .build(), + AudioFormat.Builder() + .setSampleRate(sampleRate) + .setChannelMask(AudioFormat.CHANNEL_OUT_MONO) + .setEncoding(AudioFormat.ENCODING_PCM_16BIT) + .build(), + bufferSize, + AudioTrack.MODE_STREAM, + AudioManager.AUDIO_SESSION_ID_GENERATE, + ) + if (track.state != AudioTrack.STATE_INITIALIZED) { + track.release() + throw IllegalStateException("AudioTrack init failed") + } + pcmTrack = track + track.play() + + Log.d(tag, "pcm play start sampleRate=$sampleRate bufferSize=$bufferSize") + try { + streamPcm(voiceId = voiceId, apiKey = apiKey, request = request, track = track) + } finally { + cleanupPcmTrack() + } + Log.d(tag, "pcm play done") + } + + private suspend fun speakWithSystemTts(text: String) { + val trimmed = text.trim() + if (trimmed.isEmpty()) return + val ok = ensureSystemTts() + if (!ok) { + throw IllegalStateException("system TTS unavailable") + } + + val tts = systemTts ?: throw IllegalStateException("system TTS unavailable") + val utteranceId = "talk-${UUID.randomUUID()}" + val deferred = CompletableDeferred() + systemTtsPending?.cancel() + systemTtsPending = deferred + systemTtsPendingId = utteranceId + + withContext(Dispatchers.Main) { + val params = Bundle() + tts.speak(trimmed, TextToSpeech.QUEUE_FLUSH, params, utteranceId) + } + + withContext(Dispatchers.IO) { + try { + kotlinx.coroutines.withTimeout(180_000) { deferred.await() } + } catch (err: Throwable) { + throw err + } + } + } + + private suspend fun ensureSystemTts(): Boolean { + if (systemTts != null) return true + return withContext(Dispatchers.Main) { + val deferred = CompletableDeferred() + val tts = + try { + TextToSpeech(context) { status -> + deferred.complete(status == TextToSpeech.SUCCESS) + } + } catch (_: Throwable) { + deferred.complete(false) + null + } + if (tts == null) return@withContext false + + tts.setOnUtteranceProgressListener( + object : UtteranceProgressListener() { + override fun onStart(utteranceId: String?) {} + + override fun onDone(utteranceId: String?) { + if (utteranceId == null) return + if (utteranceId != systemTtsPendingId) return + systemTtsPending?.complete(Unit) + systemTtsPending = null + systemTtsPendingId = null + } + + @Suppress("OVERRIDE_DEPRECATION") + @Deprecated("Deprecated in Java") + override fun onError(utteranceId: String?) { + if (utteranceId == null) return + if (utteranceId != systemTtsPendingId) return + systemTtsPending?.completeExceptionally(IllegalStateException("system TTS error")) + systemTtsPending = null + systemTtsPendingId = null + } + + override fun onError(utteranceId: String?, errorCode: Int) { + if (utteranceId == null) return + if (utteranceId != systemTtsPendingId) return + systemTtsPending?.completeExceptionally(IllegalStateException("system TTS error $errorCode")) + systemTtsPending = null + systemTtsPendingId = null + } + }, + ) + + val ok = + try { + deferred.await() + } catch (_: Throwable) { + false + } + if (ok) { + systemTts = tts + } else { + tts.shutdown() + } + ok + } + } + + private fun stopSpeaking(resetInterrupt: Boolean = true) { + pcmStopRequested = true + if (!_isSpeaking.value) { + cleanupPlayer() + cleanupPcmTrack() + systemTts?.stop() + systemTtsPending?.cancel() + systemTtsPending = null + systemTtsPendingId = null + return + } + if (resetInterrupt) { + val currentMs = player?.currentPosition?.toDouble() ?: 0.0 + lastInterruptedAtSeconds = currentMs / 1000.0 + } + cleanupPlayer() + cleanupPcmTrack() + systemTts?.stop() + systemTtsPending?.cancel() + systemTtsPending = null + systemTtsPendingId = null + _isSpeaking.value = false + } + + private fun cleanupPlayer() { + player?.stop() + player?.release() + player = null + streamingSource?.close() + streamingSource = null + } + + private fun cleanupPcmTrack() { + val track = pcmTrack ?: return + try { + track.pause() + track.flush() + track.stop() + } catch (_: Throwable) { + // ignore cleanup errors + } finally { + track.release() + } + pcmTrack = null + } + + private fun shouldInterrupt(transcript: String): Boolean { + val trimmed = transcript.trim() + if (trimmed.length < 3) return false + val spoken = lastSpokenText?.lowercase() + if (spoken != null && spoken.contains(trimmed.lowercase())) return false + return true + } + + private suspend fun reloadConfig() { + val envVoice = System.getenv("ELEVENLABS_VOICE_ID")?.trim() + val sagVoice = System.getenv("SAG_VOICE_ID")?.trim() + val envKey = System.getenv("ELEVENLABS_API_KEY")?.trim() + try { + val res = session.request("config.get", "{}") + val root = json.parseToJsonElement(res).asObjectOrNull() + val config = root?.get("config").asObjectOrNull() + val talk = config?.get("talk").asObjectOrNull() + val sessionCfg = config?.get("session").asObjectOrNull() + val mainKey = normalizeMainKey(sessionCfg?.get("mainKey").asStringOrNull()) + val voice = talk?.get("voiceId")?.asStringOrNull()?.trim()?.takeIf { it.isNotEmpty() } + val aliases = + talk?.get("voiceAliases").asObjectOrNull()?.entries?.mapNotNull { (key, value) -> + val id = value.asStringOrNull()?.trim()?.takeIf { it.isNotEmpty() } ?: return@mapNotNull null + normalizeAliasKey(key).takeIf { it.isNotEmpty() }?.let { it to id } + }?.toMap().orEmpty() + val model = talk?.get("modelId")?.asStringOrNull()?.trim()?.takeIf { it.isNotEmpty() } + val outputFormat = talk?.get("outputFormat")?.asStringOrNull()?.trim()?.takeIf { it.isNotEmpty() } + val key = talk?.get("apiKey")?.asStringOrNull()?.trim()?.takeIf { it.isNotEmpty() } + val interrupt = talk?.get("interruptOnSpeech")?.asBooleanOrNull() + + if (!isCanonicalMainSessionKey(mainSessionKey)) { + mainSessionKey = mainKey + } + defaultVoiceId = voice ?: envVoice?.takeIf { it.isNotEmpty() } ?: sagVoice?.takeIf { it.isNotEmpty() } + voiceAliases = aliases + if (!voiceOverrideActive) currentVoiceId = defaultVoiceId + defaultModelId = model ?: defaultModelIdFallback + if (!modelOverrideActive) currentModelId = defaultModelId + defaultOutputFormat = outputFormat ?: defaultOutputFormatFallback + apiKey = key ?: envKey?.takeIf { it.isNotEmpty() } + if (interrupt != null) interruptOnSpeech = interrupt + } catch (_: Throwable) { + defaultVoiceId = envVoice?.takeIf { it.isNotEmpty() } ?: sagVoice?.takeIf { it.isNotEmpty() } + defaultModelId = defaultModelIdFallback + if (!modelOverrideActive) currentModelId = defaultModelId + apiKey = envKey?.takeIf { it.isNotEmpty() } + voiceAliases = emptyMap() + defaultOutputFormat = defaultOutputFormatFallback + } + } + + private fun parseRunId(jsonString: String): String? { + val obj = json.parseToJsonElement(jsonString).asObjectOrNull() ?: return null + return obj["runId"].asStringOrNull() + } + + private suspend fun streamTts( + voiceId: String, + apiKey: String, + request: ElevenLabsRequest, + sink: StreamingMediaDataSource, + ) { + withContext(Dispatchers.IO) { + val conn = openTtsConnection(voiceId = voiceId, apiKey = apiKey, request = request) + try { + val payload = buildRequestPayload(request) + conn.outputStream.use { it.write(payload.toByteArray()) } + + val code = conn.responseCode + if (code >= 400) { + val message = conn.errorStream?.readBytes()?.toString(Charsets.UTF_8) ?: "" + sink.fail() + throw IllegalStateException("ElevenLabs failed: $code $message") + } + + val buffer = ByteArray(8 * 1024) + conn.inputStream.use { input -> + while (true) { + val read = input.read(buffer) + if (read <= 0) break + sink.append(buffer.copyOf(read)) + } + } + sink.finish() + } finally { + conn.disconnect() + } + } + } + + private suspend fun streamPcm( + voiceId: String, + apiKey: String, + request: ElevenLabsRequest, + track: AudioTrack, + ) { + withContext(Dispatchers.IO) { + val conn = openTtsConnection(voiceId = voiceId, apiKey = apiKey, request = request) + try { + val payload = buildRequestPayload(request) + conn.outputStream.use { it.write(payload.toByteArray()) } + + val code = conn.responseCode + if (code >= 400) { + val message = conn.errorStream?.readBytes()?.toString(Charsets.UTF_8) ?: "" + throw IllegalStateException("ElevenLabs failed: $code $message") + } + + val buffer = ByteArray(8 * 1024) + conn.inputStream.use { input -> + while (true) { + if (pcmStopRequested) return@withContext + val read = input.read(buffer) + if (read <= 0) break + var offset = 0 + while (offset < read) { + if (pcmStopRequested) return@withContext + val wrote = + try { + track.write(buffer, offset, read - offset) + } catch (err: Throwable) { + if (pcmStopRequested) return@withContext + throw err + } + if (wrote <= 0) { + if (pcmStopRequested) return@withContext + throw IllegalStateException("AudioTrack write failed: $wrote") + } + offset += wrote + } + } + } + } finally { + conn.disconnect() + } + } + } + + private fun openTtsConnection( + voiceId: String, + apiKey: String, + request: ElevenLabsRequest, + ): HttpURLConnection { + val baseUrl = "https://api.elevenlabs.io/v1/text-to-speech/$voiceId/stream" + val latencyTier = request.latencyTier + val url = + if (latencyTier != null) { + URL("$baseUrl?optimize_streaming_latency=$latencyTier") + } else { + URL(baseUrl) + } + val conn = url.openConnection() as HttpURLConnection + conn.requestMethod = "POST" + conn.connectTimeout = 30_000 + conn.readTimeout = 30_000 + conn.setRequestProperty("Content-Type", "application/json") + conn.setRequestProperty("Accept", resolveAcceptHeader(request.outputFormat)) + conn.setRequestProperty("xi-api-key", apiKey) + conn.doOutput = true + return conn + } + + private fun resolveAcceptHeader(outputFormat: String?): String { + val normalized = outputFormat?.trim()?.lowercase().orEmpty() + return if (normalized.startsWith("pcm_")) "audio/pcm" else "audio/mpeg" + } + + private fun buildRequestPayload(request: ElevenLabsRequest): String { + val voiceSettingsEntries = + buildJsonObject { + request.speed?.let { put("speed", JsonPrimitive(it)) } + request.stability?.let { put("stability", JsonPrimitive(it)) } + request.similarity?.let { put("similarity_boost", JsonPrimitive(it)) } + request.style?.let { put("style", JsonPrimitive(it)) } + request.speakerBoost?.let { put("use_speaker_boost", JsonPrimitive(it)) } + } + + val payload = + buildJsonObject { + put("text", JsonPrimitive(request.text)) + request.modelId?.takeIf { it.isNotEmpty() }?.let { put("model_id", JsonPrimitive(it)) } + request.outputFormat?.takeIf { it.isNotEmpty() }?.let { put("output_format", JsonPrimitive(it)) } + request.seed?.let { put("seed", JsonPrimitive(it)) } + request.normalize?.let { put("apply_text_normalization", JsonPrimitive(it)) } + request.language?.let { put("language_code", JsonPrimitive(it)) } + if (voiceSettingsEntries.isNotEmpty()) { + put("voice_settings", voiceSettingsEntries) + } + } + + return payload.toString() + } + + private data class ElevenLabsRequest( + val text: String, + val modelId: String?, + val outputFormat: String?, + val speed: Double?, + val stability: Double?, + val similarity: Double?, + val style: Double?, + val speakerBoost: Boolean?, + val seed: Long?, + val normalize: String?, + val language: String?, + val latencyTier: Int?, + ) + + private object TalkModeRuntime { + fun resolveSpeed(speed: Double?, rateWpm: Int?): Double? { + if (rateWpm != null && rateWpm > 0) { + val resolved = rateWpm.toDouble() / 175.0 + if (resolved <= 0.5 || resolved >= 2.0) return null + return resolved + } + if (speed != null) { + if (speed <= 0.5 || speed >= 2.0) return null + return speed + } + return null + } + + fun validatedUnit(value: Double?): Double? { + if (value == null) return null + if (value < 0 || value > 1) return null + return value + } + + fun validatedStability(value: Double?, modelId: String?): Double? { + if (value == null) return null + val normalized = modelId?.trim()?.lowercase() + if (normalized == "eleven_v3") { + return if (value == 0.0 || value == 0.5 || value == 1.0) value else null + } + return validatedUnit(value) + } + + fun validatedSeed(value: Long?): Long? { + if (value == null) return null + if (value < 0 || value > 4294967295L) return null + return value + } + + fun validatedNormalize(value: String?): String? { + val normalized = value?.trim()?.lowercase() ?: return null + return if (normalized in listOf("auto", "on", "off")) normalized else null + } + + fun validatedLanguage(value: String?): String? { + val normalized = value?.trim()?.lowercase() ?: return null + if (normalized.length != 2) return null + if (!normalized.all { it in 'a'..'z' }) return null + return normalized + } + + fun validatedOutputFormat(value: String?): String? { + val trimmed = value?.trim()?.lowercase() ?: return null + if (trimmed.isEmpty()) return null + if (trimmed.startsWith("mp3_")) return trimmed + return if (parsePcmSampleRate(trimmed) != null) trimmed else null + } + + fun validatedLatencyTier(value: Int?): Int? { + if (value == null) return null + if (value < 0 || value > 4) return null + return value + } + + fun parsePcmSampleRate(value: String?): Int? { + val trimmed = value?.trim()?.lowercase() ?: return null + if (!trimmed.startsWith("pcm_")) return null + val suffix = trimmed.removePrefix("pcm_") + val digits = suffix.takeWhile { it.isDigit() } + val rate = digits.toIntOrNull() ?: return null + return if (rate in setOf(16000, 22050, 24000, 44100)) rate else null + } + + fun isMessageTimestampAfter(timestamp: Double, sinceSeconds: Double): Boolean { + val sinceMs = sinceSeconds * 1000 + return if (timestamp > 10_000_000_000) { + timestamp >= sinceMs - 500 + } else { + timestamp >= sinceSeconds - 0.5 + } + } + } + + private fun ensureInterruptListener() { + if (!interruptOnSpeech || !_isEnabled.value) return + mainHandler.post { + if (stopRequested) return@post + if (!SpeechRecognizer.isRecognitionAvailable(context)) return@post + try { + if (recognizer == null) { + recognizer = SpeechRecognizer.createSpeechRecognizer(context).also { it.setRecognitionListener(listener) } + } + recognizer?.cancel() + startListeningInternal(markListening = false) + } catch (_: Throwable) { + // ignore + } + } + } + + private fun resolveVoiceAlias(value: String?): String? { + val trimmed = value?.trim().orEmpty() + if (trimmed.isEmpty()) return null + val normalized = normalizeAliasKey(trimmed) + voiceAliases[normalized]?.let { return it } + if (voiceAliases.values.any { it.equals(trimmed, ignoreCase = true) }) return trimmed + return if (isLikelyVoiceId(trimmed)) trimmed else null + } + + private suspend fun resolveVoiceId(preferred: String?, apiKey: String): String? { + val trimmed = preferred?.trim().orEmpty() + if (trimmed.isNotEmpty()) { + val resolved = resolveVoiceAlias(trimmed) + if (resolved != null) return resolved + Log.w(tag, "unknown voice alias $trimmed") + } + fallbackVoiceId?.let { return it } + + return try { + val voices = listVoices(apiKey) + val first = voices.firstOrNull() ?: return null + fallbackVoiceId = first.voiceId + if (defaultVoiceId.isNullOrBlank()) { + defaultVoiceId = first.voiceId + } + if (!voiceOverrideActive) { + currentVoiceId = first.voiceId + } + val name = first.name ?: "unknown" + Log.d(tag, "default voice selected $name (${first.voiceId})") + first.voiceId + } catch (err: Throwable) { + Log.w(tag, "list voices failed: ${err.message ?: err::class.simpleName}") + null + } + } + + private suspend fun listVoices(apiKey: String): List { + return withContext(Dispatchers.IO) { + val url = URL("https://api.elevenlabs.io/v1/voices") + val conn = url.openConnection() as HttpURLConnection + conn.requestMethod = "GET" + conn.connectTimeout = 15_000 + conn.readTimeout = 15_000 + conn.setRequestProperty("xi-api-key", apiKey) + + val code = conn.responseCode + val stream = if (code >= 400) conn.errorStream else conn.inputStream + val data = stream.readBytes() + if (code >= 400) { + val message = data.toString(Charsets.UTF_8) + throw IllegalStateException("ElevenLabs voices failed: $code $message") + } + + val root = json.parseToJsonElement(data.toString(Charsets.UTF_8)).asObjectOrNull() + val voices = (root?.get("voices") as? JsonArray) ?: JsonArray(emptyList()) + voices.mapNotNull { entry -> + val obj = entry.asObjectOrNull() ?: return@mapNotNull null + val voiceId = obj["voice_id"].asStringOrNull() ?: return@mapNotNull null + val name = obj["name"].asStringOrNull() + ElevenLabsVoice(voiceId, name) + } + } + } + + private fun isLikelyVoiceId(value: String): Boolean { + if (value.length < 10) return false + return value.all { it.isLetterOrDigit() || it == '-' || it == '_' } + } + + private fun normalizeAliasKey(value: String): String = + value.trim().lowercase() + + private data class ElevenLabsVoice(val voiceId: String, val name: String?) + + private val listener = + object : RecognitionListener { + override fun onReadyForSpeech(params: Bundle?) { + if (_isEnabled.value) { + _statusText.value = if (_isListening.value) "Listening" else _statusText.value + } + } + + override fun onBeginningOfSpeech() {} + + override fun onRmsChanged(rmsdB: Float) {} + + override fun onBufferReceived(buffer: ByteArray?) {} + + override fun onEndOfSpeech() { + scheduleRestart() + } + + override fun onError(error: Int) { + if (stopRequested) return + _isListening.value = false + if (error == SpeechRecognizer.ERROR_INSUFFICIENT_PERMISSIONS) { + _statusText.value = "Microphone permission required" + return + } + + _statusText.value = + when (error) { + SpeechRecognizer.ERROR_AUDIO -> "Audio error" + SpeechRecognizer.ERROR_CLIENT -> "Client error" + SpeechRecognizer.ERROR_NETWORK -> "Network error" + SpeechRecognizer.ERROR_NETWORK_TIMEOUT -> "Network timeout" + SpeechRecognizer.ERROR_NO_MATCH -> "Listening" + SpeechRecognizer.ERROR_RECOGNIZER_BUSY -> "Recognizer busy" + SpeechRecognizer.ERROR_SERVER -> "Server error" + SpeechRecognizer.ERROR_SPEECH_TIMEOUT -> "Listening" + else -> "Speech error ($error)" + } + scheduleRestart(delayMs = 600) + } + + override fun onResults(results: Bundle?) { + val list = results?.getStringArrayList(SpeechRecognizer.RESULTS_RECOGNITION).orEmpty() + list.firstOrNull()?.let { handleTranscript(it, isFinal = true) } + scheduleRestart() + } + + override fun onPartialResults(partialResults: Bundle?) { + val list = partialResults?.getStringArrayList(SpeechRecognizer.RESULTS_RECOGNITION).orEmpty() + list.firstOrNull()?.let { handleTranscript(it, isFinal = false) } + } + + override fun onEvent(eventType: Int, params: Bundle?) {} + } +} + +private fun JsonElement?.asObjectOrNull(): JsonObject? = this as? JsonObject + +private fun JsonElement?.asStringOrNull(): String? = + (this as? JsonPrimitive)?.takeIf { it.isString }?.content + +private fun JsonElement?.asDoubleOrNull(): Double? { + val primitive = this as? JsonPrimitive ?: return null + return primitive.content.toDoubleOrNull() +} + +private fun JsonElement?.asBooleanOrNull(): Boolean? { + val primitive = this as? JsonPrimitive ?: return null + val content = primitive.content.trim().lowercase() + return when (content) { + "true", "yes", "1" -> true + "false", "no", "0" -> false + else -> null + } +} diff --git a/apps/android/app/src/main/java/ai/openclaw/android/voice/VoiceWakeCommandExtractor.kt b/apps/android/app/src/main/java/ai/openclaw/android/voice/VoiceWakeCommandExtractor.kt new file mode 100644 index 0000000000000000000000000000000000000000..dccd3950c90a440359a763bca6e2fd7503c6fba9 --- /dev/null +++ b/apps/android/app/src/main/java/ai/openclaw/android/voice/VoiceWakeCommandExtractor.kt @@ -0,0 +1,40 @@ +package ai.openclaw.android.voice + +object VoiceWakeCommandExtractor { + fun extractCommand(text: String, triggerWords: List): String? { + val raw = text.trim() + if (raw.isEmpty()) return null + + val triggers = + triggerWords + .map { it.trim().lowercase() } + .filter { it.isNotEmpty() } + .distinct() + if (triggers.isEmpty()) return null + + val alternation = triggers.joinToString("|") { Regex.escape(it) } + // Match: " " + val regex = Regex("(?i)(?:^|\\s)($alternation)\\b[\\s\\p{Punct}]*([\\s\\S]+)$") + val match = regex.find(raw) ?: return null + val extracted = match.groupValues.getOrNull(2)?.trim().orEmpty() + if (extracted.isEmpty()) return null + + val cleaned = extracted.trimStart { it.isWhitespace() || it.isPunctuation() }.trim() + if (cleaned.isEmpty()) return null + return cleaned + } +} + +private fun Char.isPunctuation(): Boolean { + return when (Character.getType(this)) { + Character.CONNECTOR_PUNCTUATION.toInt(), + Character.DASH_PUNCTUATION.toInt(), + Character.START_PUNCTUATION.toInt(), + Character.END_PUNCTUATION.toInt(), + Character.INITIAL_QUOTE_PUNCTUATION.toInt(), + Character.FINAL_QUOTE_PUNCTUATION.toInt(), + Character.OTHER_PUNCTUATION.toInt(), + -> true + else -> false + } +} diff --git a/apps/android/app/src/main/java/ai/openclaw/android/voice/VoiceWakeManager.kt b/apps/android/app/src/main/java/ai/openclaw/android/voice/VoiceWakeManager.kt new file mode 100644 index 0000000000000000000000000000000000000000..334f985a02861b28a6c06a3cdeb56f45074bae8d --- /dev/null +++ b/apps/android/app/src/main/java/ai/openclaw/android/voice/VoiceWakeManager.kt @@ -0,0 +1,173 @@ +package ai.openclaw.android.voice + +import android.content.Context +import android.content.Intent +import android.os.Bundle +import android.os.Handler +import android.os.Looper +import android.speech.RecognitionListener +import android.speech.RecognizerIntent +import android.speech.SpeechRecognizer +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Job +import kotlinx.coroutines.delay +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.launch + +class VoiceWakeManager( + private val context: Context, + private val scope: CoroutineScope, + private val onCommand: suspend (String) -> Unit, +) { + private val mainHandler = Handler(Looper.getMainLooper()) + + private val _isListening = MutableStateFlow(false) + val isListening: StateFlow = _isListening + + private val _statusText = MutableStateFlow("Off") + val statusText: StateFlow = _statusText + + var triggerWords: List = emptyList() + private set + + private var recognizer: SpeechRecognizer? = null + private var restartJob: Job? = null + private var lastDispatched: String? = null + private var stopRequested = false + + fun setTriggerWords(words: List) { + triggerWords = words + } + + fun start() { + mainHandler.post { + if (_isListening.value) return@post + stopRequested = false + + if (!SpeechRecognizer.isRecognitionAvailable(context)) { + _isListening.value = false + _statusText.value = "Speech recognizer unavailable" + return@post + } + + try { + recognizer?.destroy() + recognizer = SpeechRecognizer.createSpeechRecognizer(context).also { it.setRecognitionListener(listener) } + startListeningInternal() + } catch (err: Throwable) { + _isListening.value = false + _statusText.value = "Start failed: ${err.message ?: err::class.simpleName}" + } + } + } + + fun stop(statusText: String = "Off") { + stopRequested = true + restartJob?.cancel() + restartJob = null + mainHandler.post { + _isListening.value = false + _statusText.value = statusText + recognizer?.cancel() + recognizer?.destroy() + recognizer = null + } + } + + private fun startListeningInternal() { + val r = recognizer ?: return + val intent = + Intent(RecognizerIntent.ACTION_RECOGNIZE_SPEECH).apply { + putExtra(RecognizerIntent.EXTRA_LANGUAGE_MODEL, RecognizerIntent.LANGUAGE_MODEL_FREE_FORM) + putExtra(RecognizerIntent.EXTRA_PARTIAL_RESULTS, true) + putExtra(RecognizerIntent.EXTRA_MAX_RESULTS, 3) + putExtra(RecognizerIntent.EXTRA_CALLING_PACKAGE, context.packageName) + } + + _statusText.value = "Listening" + _isListening.value = true + r.startListening(intent) + } + + private fun scheduleRestart(delayMs: Long = 350) { + if (stopRequested) return + restartJob?.cancel() + restartJob = + scope.launch { + delay(delayMs) + mainHandler.post { + if (stopRequested) return@post + try { + recognizer?.cancel() + startListeningInternal() + } catch (_: Throwable) { + // Will be picked up by onError and retry again. + } + } + } + } + + private fun handleTranscription(text: String) { + val command = VoiceWakeCommandExtractor.extractCommand(text, triggerWords) ?: return + if (command == lastDispatched) return + lastDispatched = command + + scope.launch { onCommand(command) } + _statusText.value = "Triggered" + scheduleRestart(delayMs = 650) + } + + private val listener = + object : RecognitionListener { + override fun onReadyForSpeech(params: Bundle?) { + _statusText.value = "Listening" + } + + override fun onBeginningOfSpeech() {} + + override fun onRmsChanged(rmsdB: Float) {} + + override fun onBufferReceived(buffer: ByteArray?) {} + + override fun onEndOfSpeech() { + scheduleRestart() + } + + override fun onError(error: Int) { + if (stopRequested) return + _isListening.value = false + if (error == SpeechRecognizer.ERROR_INSUFFICIENT_PERMISSIONS) { + _statusText.value = "Microphone permission required" + return + } + + _statusText.value = + when (error) { + SpeechRecognizer.ERROR_AUDIO -> "Audio error" + SpeechRecognizer.ERROR_CLIENT -> "Client error" + SpeechRecognizer.ERROR_NETWORK -> "Network error" + SpeechRecognizer.ERROR_NETWORK_TIMEOUT -> "Network timeout" + SpeechRecognizer.ERROR_NO_MATCH -> "Listening" + SpeechRecognizer.ERROR_RECOGNIZER_BUSY -> "Recognizer busy" + SpeechRecognizer.ERROR_SERVER -> "Server error" + SpeechRecognizer.ERROR_SPEECH_TIMEOUT -> "Listening" + else -> "Speech error ($error)" + } + scheduleRestart(delayMs = 600) + } + + override fun onResults(results: Bundle?) { + val list = results?.getStringArrayList(SpeechRecognizer.RESULTS_RECOGNITION).orEmpty() + list.firstOrNull()?.let(::handleTranscription) + scheduleRestart() + } + + override fun onPartialResults(partialResults: Bundle?) { + val list = partialResults?.getStringArrayList(SpeechRecognizer.RESULTS_RECOGNITION).orEmpty() + list.firstOrNull()?.let(::handleTranscription) + } + + override fun onEvent(eventType: Int, params: Bundle?) {} + } +} diff --git a/apps/android/app/src/main/res/mipmap-anydpi/ic_launcher.xml b/apps/android/app/src/main/res/mipmap-anydpi/ic_launcher.xml new file mode 100644 index 0000000000000000000000000000000000000000..6f379984a93ed64a59b254b6ead0369698fe842a --- /dev/null +++ b/apps/android/app/src/main/res/mipmap-anydpi/ic_launcher.xml @@ -0,0 +1,6 @@ + + + + + + diff --git a/apps/android/app/src/main/res/mipmap-anydpi/ic_launcher_round.xml b/apps/android/app/src/main/res/mipmap-anydpi/ic_launcher_round.xml new file mode 100644 index 0000000000000000000000000000000000000000..6f379984a93ed64a59b254b6ead0369698fe842a --- /dev/null +++ b/apps/android/app/src/main/res/mipmap-anydpi/ic_launcher_round.xml @@ -0,0 +1,6 @@ + + + + + + diff --git a/apps/android/app/src/main/res/mipmap-hdpi/ic_launcher.png b/apps/android/app/src/main/res/mipmap-hdpi/ic_launcher.png new file mode 100644 index 0000000000000000000000000000000000000000..613e26663836ac2bb973b6af97d0072e5469fd01 Binary files /dev/null and b/apps/android/app/src/main/res/mipmap-hdpi/ic_launcher.png differ diff --git a/apps/android/app/src/main/res/mipmap-hdpi/ic_launcher_foreground.png b/apps/android/app/src/main/res/mipmap-hdpi/ic_launcher_foreground.png new file mode 100644 index 0000000000000000000000000000000000000000..22442bc1d80373b884c11241ab136536e8215e7c Binary files /dev/null and b/apps/android/app/src/main/res/mipmap-hdpi/ic_launcher_foreground.png differ diff --git a/apps/android/app/src/main/res/mipmap-mdpi/ic_launcher.png b/apps/android/app/src/main/res/mipmap-mdpi/ic_launcher.png new file mode 100644 index 0000000000000000000000000000000000000000..b1fd747de01176d1891e89132524fb6f42665fd1 Binary files /dev/null and b/apps/android/app/src/main/res/mipmap-mdpi/ic_launcher.png differ diff --git a/apps/android/app/src/main/res/mipmap-mdpi/ic_launcher_foreground.png b/apps/android/app/src/main/res/mipmap-mdpi/ic_launcher_foreground.png new file mode 100644 index 0000000000000000000000000000000000000000..d26c0189852d3e650da64609a593c897b1f9d4ca Binary files /dev/null and b/apps/android/app/src/main/res/mipmap-mdpi/ic_launcher_foreground.png differ diff --git a/apps/android/app/src/main/res/mipmap-xhdpi/ic_launcher.png b/apps/android/app/src/main/res/mipmap-xhdpi/ic_launcher.png new file mode 100644 index 0000000000000000000000000000000000000000..038e3dc7a7091109da0b257c830a0e85a078c4cb Binary files /dev/null and b/apps/android/app/src/main/res/mipmap-xhdpi/ic_launcher.png differ diff --git a/apps/android/app/src/main/res/mipmap-xhdpi/ic_launcher_foreground.png b/apps/android/app/src/main/res/mipmap-xhdpi/ic_launcher_foreground.png new file mode 100644 index 0000000000000000000000000000000000000000..2f06597022559b50eeb14e4505f694ea68d4a211 Binary files /dev/null and b/apps/android/app/src/main/res/mipmap-xhdpi/ic_launcher_foreground.png differ diff --git a/apps/android/app/src/main/res/mipmap-xxhdpi/ic_launcher.png b/apps/android/app/src/main/res/mipmap-xxhdpi/ic_launcher.png new file mode 100644 index 0000000000000000000000000000000000000000..a5d995c2ee236b987512e04d4c77bb147d648824 Binary files /dev/null and b/apps/android/app/src/main/res/mipmap-xxhdpi/ic_launcher.png differ diff --git a/apps/android/app/src/main/res/mipmap-xxhdpi/ic_launcher_foreground.png b/apps/android/app/src/main/res/mipmap-xxhdpi/ic_launcher_foreground.png new file mode 100644 index 0000000000000000000000000000000000000000..8a49e8d7feb57ded1a53a0e754205a9dfd25b431 --- /dev/null +++ b/apps/android/app/src/main/res/mipmap-xxhdpi/ic_launcher_foreground.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d0cbccef60edf0799d586d21c04fa90a7aac70b04f628dc03778463fa12ff453 +size 158927 diff --git a/apps/android/app/src/main/res/mipmap-xxxhdpi/ic_launcher.png b/apps/android/app/src/main/res/mipmap-xxxhdpi/ic_launcher.png new file mode 100644 index 0000000000000000000000000000000000000000..ceabff1f5626ea98dec97ecaa017ff2b27e6fe8b Binary files /dev/null and b/apps/android/app/src/main/res/mipmap-xxxhdpi/ic_launcher.png differ diff --git a/apps/android/app/src/main/res/mipmap-xxxhdpi/ic_launcher_foreground.png b/apps/android/app/src/main/res/mipmap-xxxhdpi/ic_launcher_foreground.png new file mode 100644 index 0000000000000000000000000000000000000000..311646e71fa4f4e45d8ebe404a8c71b91df3eeae --- /dev/null +++ b/apps/android/app/src/main/res/mipmap-xxxhdpi/ic_launcher_foreground.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c82366aad8cf36b88131c365315e92b64ec6bb78f20bd16de3f20b51fb7ec513 +size 276047 diff --git a/apps/android/app/src/main/res/values/colors.xml b/apps/android/app/src/main/res/values/colors.xml new file mode 100644 index 0000000000000000000000000000000000000000..dfadc94cf0354372f86f1c446ea8df37b7c89c5a --- /dev/null +++ b/apps/android/app/src/main/res/values/colors.xml @@ -0,0 +1,3 @@ + + #0A0A0A + diff --git a/apps/android/app/src/main/res/values/strings.xml b/apps/android/app/src/main/res/values/strings.xml new file mode 100644 index 0000000000000000000000000000000000000000..0098cee20f0e3e8d76380595ff822a6bc2611604 --- /dev/null +++ b/apps/android/app/src/main/res/values/strings.xml @@ -0,0 +1,3 @@ + + OpenClaw Node + diff --git a/apps/android/app/src/main/res/values/themes.xml b/apps/android/app/src/main/res/values/themes.xml new file mode 100644 index 0000000000000000000000000000000000000000..3ac5d04d83183fc31a19020c6a2015a1c481ad33 --- /dev/null +++ b/apps/android/app/src/main/res/values/themes.xml @@ -0,0 +1,7 @@ + + + diff --git a/apps/android/app/src/main/res/xml/backup_rules.xml b/apps/android/app/src/main/res/xml/backup_rules.xml new file mode 100644 index 0000000000000000000000000000000000000000..21e592ca47aceeacfc6e33489bddd938fc2a6b01 --- /dev/null +++ b/apps/android/app/src/main/res/xml/backup_rules.xml @@ -0,0 +1,4 @@ + + + + diff --git a/apps/android/app/src/main/res/xml/data_extraction_rules.xml b/apps/android/app/src/main/res/xml/data_extraction_rules.xml new file mode 100644 index 0000000000000000000000000000000000000000..46e58c54eb07332ef788ff0d1e661e03dc119f00 --- /dev/null +++ b/apps/android/app/src/main/res/xml/data_extraction_rules.xml @@ -0,0 +1,9 @@ + + + + + + + + + diff --git a/apps/android/app/src/main/res/xml/network_security_config.xml b/apps/android/app/src/main/res/xml/network_security_config.xml new file mode 100644 index 0000000000000000000000000000000000000000..7ac5f5cdd7ba4a7a97a22b59f2890825068d0388 --- /dev/null +++ b/apps/android/app/src/main/res/xml/network_security_config.xml @@ -0,0 +1,12 @@ + + + + + + + openclaw.local + + + ts.net + + diff --git a/apps/android/app/src/test/java/ai/openclaw/android/NodeForegroundServiceTest.kt b/apps/android/app/src/test/java/ai/openclaw/android/NodeForegroundServiceTest.kt new file mode 100644 index 0000000000000000000000000000000000000000..7a81936ecd2ea9bc8366a60262bfe20fc8939f41 --- /dev/null +++ b/apps/android/app/src/test/java/ai/openclaw/android/NodeForegroundServiceTest.kt @@ -0,0 +1,43 @@ +package ai.openclaw.android + +import android.app.Notification +import android.content.Intent +import org.junit.Assert.assertEquals +import org.junit.Assert.assertNotNull +import org.junit.Test +import org.junit.runner.RunWith +import org.robolectric.Robolectric +import org.robolectric.RobolectricTestRunner +import org.robolectric.Shadows +import org.robolectric.annotation.Config + +@RunWith(RobolectricTestRunner::class) +@Config(sdk = [34]) +class NodeForegroundServiceTest { + @Test + fun buildNotificationSetsLaunchIntent() { + val service = Robolectric.buildService(NodeForegroundService::class.java).get() + val notification = buildNotification(service) + + val pendingIntent = notification.contentIntent + assertNotNull(pendingIntent) + + val savedIntent = Shadows.shadowOf(pendingIntent).savedIntent + assertNotNull(savedIntent) + assertEquals(MainActivity::class.java.name, savedIntent.component?.className) + + val expectedFlags = Intent.FLAG_ACTIVITY_SINGLE_TOP or Intent.FLAG_ACTIVITY_CLEAR_TOP + assertEquals(expectedFlags, savedIntent.flags and expectedFlags) + } + + private fun buildNotification(service: NodeForegroundService): Notification { + val method = + NodeForegroundService::class.java.getDeclaredMethod( + "buildNotification", + String::class.java, + String::class.java, + ) + method.isAccessible = true + return method.invoke(service, "Title", "Text") as Notification + } +} diff --git a/apps/android/app/src/test/java/ai/openclaw/android/WakeWordsTest.kt b/apps/android/app/src/test/java/ai/openclaw/android/WakeWordsTest.kt new file mode 100644 index 0000000000000000000000000000000000000000..55730e2f5ab9137b517d5e5a7a8b3071bdbc99f5 --- /dev/null +++ b/apps/android/app/src/test/java/ai/openclaw/android/WakeWordsTest.kt @@ -0,0 +1,50 @@ +package ai.openclaw.android + +import org.junit.Assert.assertEquals +import org.junit.Assert.assertNull +import org.junit.Test + +class WakeWordsTest { + @Test + fun parseCommaSeparatedTrimsAndDropsEmpty() { + assertEquals(listOf("openclaw", "claude"), WakeWords.parseCommaSeparated(" openclaw , claude, , ")) + } + + @Test + fun sanitizeTrimsCapsAndFallsBack() { + val defaults = listOf("openclaw", "claude") + val long = "x".repeat(WakeWords.maxWordLength + 10) + val words = listOf(" ", " hello ", long) + + val sanitized = WakeWords.sanitize(words, defaults) + assertEquals(2, sanitized.size) + assertEquals("hello", sanitized[0]) + assertEquals("x".repeat(WakeWords.maxWordLength), sanitized[1]) + + assertEquals(defaults, WakeWords.sanitize(listOf(" ", ""), defaults)) + } + + @Test + fun sanitizeLimitsWordCount() { + val defaults = listOf("openclaw") + val words = (1..(WakeWords.maxWords + 5)).map { "w$it" } + val sanitized = WakeWords.sanitize(words, defaults) + assertEquals(WakeWords.maxWords, sanitized.size) + assertEquals("w1", sanitized.first()) + assertEquals("w${WakeWords.maxWords}", sanitized.last()) + } + + @Test + fun parseIfChangedSkipsWhenUnchanged() { + val current = listOf("openclaw", "claude") + val parsed = WakeWords.parseIfChanged(" openclaw , claude ", current) + assertNull(parsed) + } + + @Test + fun parseIfChangedReturnsUpdatedList() { + val current = listOf("openclaw") + val parsed = WakeWords.parseIfChanged(" openclaw , jarvis ", current) + assertEquals(listOf("openclaw", "jarvis"), parsed) + } +} diff --git a/apps/android/app/src/test/java/ai/openclaw/android/gateway/BonjourEscapesTest.kt b/apps/android/app/src/test/java/ai/openclaw/android/gateway/BonjourEscapesTest.kt new file mode 100644 index 0000000000000000000000000000000000000000..fe00e50a72dd808d0ed281c1f375e1b7f263ed40 --- /dev/null +++ b/apps/android/app/src/test/java/ai/openclaw/android/gateway/BonjourEscapesTest.kt @@ -0,0 +1,19 @@ +package ai.openclaw.android.gateway + +import org.junit.Assert.assertEquals +import org.junit.Test + +class BonjourEscapesTest { + @Test + fun decodeNoop() { + assertEquals("", BonjourEscapes.decode("")) + assertEquals("hello", BonjourEscapes.decode("hello")) + } + + @Test + fun decodeDecodesDecimalEscapes() { + assertEquals("OpenClaw Gateway", BonjourEscapes.decode("OpenClaw\\032Gateway")) + assertEquals("A B", BonjourEscapes.decode("A\\032B")) + assertEquals("Peter\u2019s Mac", BonjourEscapes.decode("Peter\\226\\128\\153s Mac")) + } +} diff --git a/apps/android/app/src/test/java/ai/openclaw/android/node/CanvasControllerSnapshotParamsTest.kt b/apps/android/app/src/test/java/ai/openclaw/android/node/CanvasControllerSnapshotParamsTest.kt new file mode 100644 index 0000000000000000000000000000000000000000..dd1b9d5d19ab5ff6e09233dde2d17d19c5c7acf1 --- /dev/null +++ b/apps/android/app/src/test/java/ai/openclaw/android/node/CanvasControllerSnapshotParamsTest.kt @@ -0,0 +1,43 @@ +package ai.openclaw.android.node + +import org.junit.Assert.assertEquals +import org.junit.Assert.assertNull +import org.junit.Test + +class CanvasControllerSnapshotParamsTest { + @Test + fun parseSnapshotParamsDefaultsToJpeg() { + val params = CanvasController.parseSnapshotParams(null) + assertEquals(CanvasController.SnapshotFormat.Jpeg, params.format) + assertNull(params.quality) + assertNull(params.maxWidth) + } + + @Test + fun parseSnapshotParamsParsesPng() { + val params = CanvasController.parseSnapshotParams("""{"format":"png","maxWidth":900}""") + assertEquals(CanvasController.SnapshotFormat.Png, params.format) + assertEquals(900, params.maxWidth) + } + + @Test + fun parseSnapshotParamsParsesJpegAliases() { + assertEquals( + CanvasController.SnapshotFormat.Jpeg, + CanvasController.parseSnapshotParams("""{"format":"jpeg"}""").format, + ) + assertEquals( + CanvasController.SnapshotFormat.Jpeg, + CanvasController.parseSnapshotParams("""{"format":"jpg"}""").format, + ) + } + + @Test + fun parseSnapshotParamsClampsQuality() { + val low = CanvasController.parseSnapshotParams("""{"quality":0.01}""") + assertEquals(0.1, low.quality) + + val high = CanvasController.parseSnapshotParams("""{"quality":5}""") + assertEquals(1.0, high.quality) + } +} diff --git a/apps/android/app/src/test/java/ai/openclaw/android/node/JpegSizeLimiterTest.kt b/apps/android/app/src/test/java/ai/openclaw/android/node/JpegSizeLimiterTest.kt new file mode 100644 index 0000000000000000000000000000000000000000..5de1dd5451a016d2be0d618141b69f5de6bb12cc --- /dev/null +++ b/apps/android/app/src/test/java/ai/openclaw/android/node/JpegSizeLimiterTest.kt @@ -0,0 +1,47 @@ +package ai.openclaw.android.node + +import org.junit.Assert.assertEquals +import org.junit.Assert.assertTrue +import org.junit.Test +import kotlin.math.min + +class JpegSizeLimiterTest { + @Test + fun compressesLargePayloadsUnderLimit() { + val maxBytes = 5 * 1024 * 1024 + val result = + JpegSizeLimiter.compressToLimit( + initialWidth = 4000, + initialHeight = 3000, + startQuality = 95, + maxBytes = maxBytes, + encode = { width, height, quality -> + val estimated = (width.toLong() * height.toLong() * quality.toLong()) / 100 + val size = min(maxBytes.toLong() * 2, estimated).toInt() + ByteArray(size) + }, + ) + + assertTrue(result.bytes.size <= maxBytes) + assertTrue(result.width <= 4000) + assertTrue(result.height <= 3000) + assertTrue(result.quality <= 95) + } + + @Test + fun keepsSmallPayloadsAsIs() { + val maxBytes = 5 * 1024 * 1024 + val result = + JpegSizeLimiter.compressToLimit( + initialWidth = 800, + initialHeight = 600, + startQuality = 90, + maxBytes = maxBytes, + encode = { _, _, _ -> ByteArray(120_000) }, + ) + + assertEquals(800, result.width) + assertEquals(600, result.height) + assertEquals(90, result.quality) + } +} diff --git a/apps/android/app/src/test/java/ai/openclaw/android/node/SmsManagerTest.kt b/apps/android/app/src/test/java/ai/openclaw/android/node/SmsManagerTest.kt new file mode 100644 index 0000000000000000000000000000000000000000..a3d61329b4a1044bae619bcae7ceb3bd98a92bf3 --- /dev/null +++ b/apps/android/app/src/test/java/ai/openclaw/android/node/SmsManagerTest.kt @@ -0,0 +1,91 @@ +package ai.openclaw.android.node + +import kotlinx.serialization.json.jsonObject +import kotlinx.serialization.json.jsonPrimitive +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertTrue +import org.junit.Test + +class SmsManagerTest { + private val json = SmsManager.JsonConfig + + @Test + fun parseParamsRejectsEmptyPayload() { + val result = SmsManager.parseParams("", json) + assertTrue(result is SmsManager.ParseResult.Error) + val error = result as SmsManager.ParseResult.Error + assertEquals("INVALID_REQUEST: paramsJSON required", error.error) + } + + @Test + fun parseParamsRejectsInvalidJson() { + val result = SmsManager.parseParams("not-json", json) + assertTrue(result is SmsManager.ParseResult.Error) + val error = result as SmsManager.ParseResult.Error + assertEquals("INVALID_REQUEST: expected JSON object", error.error) + } + + @Test + fun parseParamsRejectsNonObjectJson() { + val result = SmsManager.parseParams("[]", json) + assertTrue(result is SmsManager.ParseResult.Error) + val error = result as SmsManager.ParseResult.Error + assertEquals("INVALID_REQUEST: expected JSON object", error.error) + } + + @Test + fun parseParamsRejectsMissingTo() { + val result = SmsManager.parseParams("{\"message\":\"Hi\"}", json) + assertTrue(result is SmsManager.ParseResult.Error) + val error = result as SmsManager.ParseResult.Error + assertEquals("INVALID_REQUEST: 'to' phone number required", error.error) + assertEquals("Hi", error.message) + } + + @Test + fun parseParamsRejectsMissingMessage() { + val result = SmsManager.parseParams("{\"to\":\"+1234\"}", json) + assertTrue(result is SmsManager.ParseResult.Error) + val error = result as SmsManager.ParseResult.Error + assertEquals("INVALID_REQUEST: 'message' text required", error.error) + assertEquals("+1234", error.to) + } + + @Test + fun parseParamsTrimsToField() { + val result = SmsManager.parseParams("{\"to\":\" +1555 \",\"message\":\"Hello\"}", json) + assertTrue(result is SmsManager.ParseResult.Ok) + val ok = result as SmsManager.ParseResult.Ok + assertEquals("+1555", ok.params.to) + assertEquals("Hello", ok.params.message) + } + + @Test + fun buildPayloadJsonEscapesFields() { + val payload = SmsManager.buildPayloadJson( + json = json, + ok = false, + to = "+1\"23", + error = "SMS_SEND_FAILED: \"nope\"", + ) + val parsed = json.parseToJsonElement(payload).jsonObject + assertEquals("false", parsed["ok"]?.jsonPrimitive?.content) + assertEquals("+1\"23", parsed["to"]?.jsonPrimitive?.content) + assertEquals("SMS_SEND_FAILED: \"nope\"", parsed["error"]?.jsonPrimitive?.content) + } + + @Test + fun buildSendPlanUsesMultipartWhenMultipleParts() { + val plan = SmsManager.buildSendPlan("hello") { listOf("a", "b") } + assertTrue(plan.useMultipart) + assertEquals(listOf("a", "b"), plan.parts) + } + + @Test + fun buildSendPlanFallsBackToSinglePartWhenDividerEmpty() { + val plan = SmsManager.buildSendPlan("hello") { emptyList() } + assertFalse(plan.useMultipart) + assertEquals(listOf("hello"), plan.parts) + } +} diff --git a/apps/android/app/src/test/java/ai/openclaw/android/protocol/OpenClawCanvasA2UIActionTest.kt b/apps/android/app/src/test/java/ai/openclaw/android/protocol/OpenClawCanvasA2UIActionTest.kt new file mode 100644 index 0000000000000000000000000000000000000000..c767d2eb910f7fa3d3c351fbcf96d85d1706790e --- /dev/null +++ b/apps/android/app/src/test/java/ai/openclaw/android/protocol/OpenClawCanvasA2UIActionTest.kt @@ -0,0 +1,49 @@ +package ai.openclaw.android.protocol + +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.jsonObject +import org.junit.Assert.assertEquals +import org.junit.Test + +class OpenClawCanvasA2UIActionTest { + @Test + fun extractActionNameAcceptsNameOrAction() { + val nameObj = Json.parseToJsonElement("{\"name\":\"Hello\"}").jsonObject + assertEquals("Hello", OpenClawCanvasA2UIAction.extractActionName(nameObj)) + + val actionObj = Json.parseToJsonElement("{\"action\":\"Wave\"}").jsonObject + assertEquals("Wave", OpenClawCanvasA2UIAction.extractActionName(actionObj)) + + val fallbackObj = + Json.parseToJsonElement("{\"name\":\" \",\"action\":\"Fallback\"}").jsonObject + assertEquals("Fallback", OpenClawCanvasA2UIAction.extractActionName(fallbackObj)) + } + + @Test + fun formatAgentMessageMatchesSharedSpec() { + val msg = + OpenClawCanvasA2UIAction.formatAgentMessage( + actionName = "Get Weather", + sessionKey = "main", + surfaceId = "main", + sourceComponentId = "btnWeather", + host = "Peter’s iPad", + instanceId = "ipad16,6", + contextJson = "{\"city\":\"Vienna\"}", + ) + + assertEquals( + "CANVAS_A2UI action=Get_Weather session=main surface=main component=btnWeather host=Peter_s_iPad instance=ipad16_6 ctx={\"city\":\"Vienna\"} default=update_canvas", + msg, + ) + } + + @Test + fun jsDispatchA2uiStatusIsStable() { + val js = OpenClawCanvasA2UIAction.jsDispatchA2UIActionStatus(actionId = "a1", ok = true, error = null) + assertEquals( + "window.dispatchEvent(new CustomEvent('openclaw:a2ui-action-status', { detail: { id: \"a1\", ok: true, error: \"\" } }));", + js, + ) + } +} diff --git a/apps/android/app/src/test/java/ai/openclaw/android/protocol/OpenClawProtocolConstantsTest.kt b/apps/android/app/src/test/java/ai/openclaw/android/protocol/OpenClawProtocolConstantsTest.kt new file mode 100644 index 0000000000000000000000000000000000000000..10ab733ae53f184f0e4e7e5e635e145c38c8046d --- /dev/null +++ b/apps/android/app/src/test/java/ai/openclaw/android/protocol/OpenClawProtocolConstantsTest.kt @@ -0,0 +1,35 @@ +package ai.openclaw.android.protocol + +import org.junit.Assert.assertEquals +import org.junit.Test + +class OpenClawProtocolConstantsTest { + @Test + fun canvasCommandsUseStableStrings() { + assertEquals("canvas.present", OpenClawCanvasCommand.Present.rawValue) + assertEquals("canvas.hide", OpenClawCanvasCommand.Hide.rawValue) + assertEquals("canvas.navigate", OpenClawCanvasCommand.Navigate.rawValue) + assertEquals("canvas.eval", OpenClawCanvasCommand.Eval.rawValue) + assertEquals("canvas.snapshot", OpenClawCanvasCommand.Snapshot.rawValue) + } + + @Test + fun a2uiCommandsUseStableStrings() { + assertEquals("canvas.a2ui.push", OpenClawCanvasA2UICommand.Push.rawValue) + assertEquals("canvas.a2ui.pushJSONL", OpenClawCanvasA2UICommand.PushJSONL.rawValue) + assertEquals("canvas.a2ui.reset", OpenClawCanvasA2UICommand.Reset.rawValue) + } + + @Test + fun capabilitiesUseStableStrings() { + assertEquals("canvas", OpenClawCapability.Canvas.rawValue) + assertEquals("camera", OpenClawCapability.Camera.rawValue) + assertEquals("screen", OpenClawCapability.Screen.rawValue) + assertEquals("voiceWake", OpenClawCapability.VoiceWake.rawValue) + } + + @Test + fun screenCommandsUseStableStrings() { + assertEquals("screen.record", OpenClawScreenCommand.Record.rawValue) + } +} diff --git a/apps/android/app/src/test/java/ai/openclaw/android/ui/chat/SessionFiltersTest.kt b/apps/android/app/src/test/java/ai/openclaw/android/ui/chat/SessionFiltersTest.kt new file mode 100644 index 0000000000000000000000000000000000000000..8e9e58000958943a4dbe405a2a135e3ba4b858f2 --- /dev/null +++ b/apps/android/app/src/test/java/ai/openclaw/android/ui/chat/SessionFiltersTest.kt @@ -0,0 +1,35 @@ +package ai.openclaw.android.ui.chat + +import ai.openclaw.android.chat.ChatSessionEntry +import org.junit.Assert.assertEquals +import org.junit.Test + +class SessionFiltersTest { + @Test + fun sessionChoicesPreferMainAndRecent() { + val now = 1_700_000_000_000L + val recent1 = now - 2 * 60 * 60 * 1000L + val recent2 = now - 5 * 60 * 60 * 1000L + val stale = now - 26 * 60 * 60 * 1000L + val sessions = + listOf( + ChatSessionEntry(key = "recent-1", updatedAtMs = recent1), + ChatSessionEntry(key = "main", updatedAtMs = stale), + ChatSessionEntry(key = "old-1", updatedAtMs = stale), + ChatSessionEntry(key = "recent-2", updatedAtMs = recent2), + ) + + val result = resolveSessionChoices("main", sessions, mainSessionKey = "main", nowMs = now).map { it.key } + assertEquals(listOf("main", "recent-1", "recent-2"), result) + } + + @Test + fun sessionChoicesIncludeCurrentWhenMissing() { + val now = 1_700_000_000_000L + val recent = now - 10 * 60 * 1000L + val sessions = listOf(ChatSessionEntry(key = "main", updatedAtMs = recent)) + + val result = resolveSessionChoices("custom", sessions, mainSessionKey = "main", nowMs = now).map { it.key } + assertEquals(listOf("main", "custom"), result) + } +} diff --git a/apps/android/app/src/test/java/ai/openclaw/android/voice/TalkDirectiveParserTest.kt b/apps/android/app/src/test/java/ai/openclaw/android/voice/TalkDirectiveParserTest.kt new file mode 100644 index 0000000000000000000000000000000000000000..77d62849c6c155d2005d649f3d743689376edc1f --- /dev/null +++ b/apps/android/app/src/test/java/ai/openclaw/android/voice/TalkDirectiveParserTest.kt @@ -0,0 +1,55 @@ +package ai.openclaw.android.voice + +import org.junit.Assert.assertEquals +import org.junit.Assert.assertNull +import org.junit.Assert.assertTrue +import org.junit.Test + +class TalkDirectiveParserTest { + @Test + fun parsesDirectiveAndStripsHeader() { + val input = """ + {"voice":"voice-123","once":true} + Hello from talk mode. + """.trimIndent() + val result = TalkDirectiveParser.parse(input) + assertEquals("voice-123", result.directive?.voiceId) + assertEquals(true, result.directive?.once) + assertEquals("Hello from talk mode.", result.stripped.trim()) + } + + @Test + fun ignoresUnknownKeysButReportsThem() { + val input = """ + {"voice":"abc","foo":1,"bar":"baz"} + Hi there. + """.trimIndent() + val result = TalkDirectiveParser.parse(input) + assertEquals("abc", result.directive?.voiceId) + assertTrue(result.unknownKeys.containsAll(listOf("bar", "foo"))) + } + + @Test + fun parsesAlternateKeys() { + val input = """ + {"model_id":"eleven_v3","similarity_boost":0.4,"no_speaker_boost":true,"rate":200} + Speak. + """.trimIndent() + val result = TalkDirectiveParser.parse(input) + assertEquals("eleven_v3", result.directive?.modelId) + assertEquals(0.4, result.directive?.similarity) + assertEquals(false, result.directive?.speakerBoost) + assertEquals(200, result.directive?.rateWpm) + } + + @Test + fun returnsNullWhenNoDirectivePresent() { + val input = """ + {} + Hello. + """.trimIndent() + val result = TalkDirectiveParser.parse(input) + assertNull(result.directive) + assertEquals(input, result.stripped) + } +} diff --git a/apps/android/app/src/test/java/ai/openclaw/android/voice/VoiceWakeCommandExtractorTest.kt b/apps/android/app/src/test/java/ai/openclaw/android/voice/VoiceWakeCommandExtractorTest.kt new file mode 100644 index 0000000000000000000000000000000000000000..76b50d8abcd0fe54e2e1096adc1c353be21cd9c8 --- /dev/null +++ b/apps/android/app/src/test/java/ai/openclaw/android/voice/VoiceWakeCommandExtractorTest.kt @@ -0,0 +1,25 @@ +package ai.openclaw.android.voice + +import org.junit.Assert.assertEquals +import org.junit.Assert.assertNull +import org.junit.Test + +class VoiceWakeCommandExtractorTest { + @Test + fun extractsCommandAfterTriggerWord() { + val res = VoiceWakeCommandExtractor.extractCommand("Claude take a photo", listOf("openclaw", "claude")) + assertEquals("take a photo", res) + } + + @Test + fun extractsCommandWithPunctuation() { + val res = VoiceWakeCommandExtractor.extractCommand("hey openclaw, what's the weather?", listOf("openclaw")) + assertEquals("what's the weather?", res) + } + + @Test + fun returnsNullWhenNoCommandProvided() { + assertNull(VoiceWakeCommandExtractor.extractCommand("claude", listOf("claude"))) + assertNull(VoiceWakeCommandExtractor.extractCommand("hey claude!", listOf("claude"))) + } +} diff --git a/apps/android/build.gradle.kts b/apps/android/build.gradle.kts new file mode 100644 index 0000000000000000000000000000000000000000..f79902d5615f60ede1c627d8039ea76c87daf1be --- /dev/null +++ b/apps/android/build.gradle.kts @@ -0,0 +1,6 @@ +plugins { + id("com.android.application") version "8.13.2" apply false + id("org.jetbrains.kotlin.android") version "2.2.21" apply false + id("org.jetbrains.kotlin.plugin.compose") version "2.2.21" apply false + id("org.jetbrains.kotlin.plugin.serialization") version "2.2.21" apply false +} diff --git a/apps/android/gradle.properties b/apps/android/gradle.properties new file mode 100644 index 0000000000000000000000000000000000000000..0742f09d58e03a095ea80f28e8db50735679da7c --- /dev/null +++ b/apps/android/gradle.properties @@ -0,0 +1,4 @@ +org.gradle.jvmargs=-Xmx3g -Dfile.encoding=UTF-8 --enable-native-access=ALL-UNNAMED +org.gradle.warning.mode=none +android.useAndroidX=true +android.nonTransitiveRClass=true diff --git a/apps/android/gradle/wrapper/gradle-wrapper.jar b/apps/android/gradle/wrapper/gradle-wrapper.jar new file mode 100644 index 0000000000000000000000000000000000000000..e6441136f3d4ba8a0da8d277868979cfbc8ad796 Binary files /dev/null and b/apps/android/gradle/wrapper/gradle-wrapper.jar differ diff --git a/apps/android/gradle/wrapper/gradle-wrapper.properties b/apps/android/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 0000000000000000000000000000000000000000..23449a2b54328b514d1423812706edf212fa587b --- /dev/null +++ b/apps/android/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,7 @@ +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-9.2.1-bin.zip +networkTimeout=10000 +validateDistributionUrl=true +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists diff --git a/apps/android/gradlew b/apps/android/gradlew new file mode 100644 index 0000000000000000000000000000000000000000..6e5806dcc2481e6ec2a8af2905aaa4c8e407a161 --- /dev/null +++ b/apps/android/gradlew @@ -0,0 +1,249 @@ +#!/bin/sh + +# +# Copyright © 2015-2021 the original authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +############################################################################## +# +# Gradle start up script for POSIX generated by Gradle. +# +# Important for running: +# +# (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is +# noncompliant, but you have some other compliant shell such as ksh or +# bash, then to run this script, type that shell name before the whole +# command line, like: +# +# ksh Gradle +# +# Busybox and similar reduced shells will NOT work, because this script +# requires all of these POSIX shell features: +# * functions; +# * expansions «$var», «${var}», «${var:-default}», «${var+SET}», +# «${var#prefix}», «${var%suffix}», and «$( cmd )»; +# * compound commands having a testable exit status, especially «case»; +# * various built-in commands including «command», «set», and «ulimit». +# +# Important for patching: +# +# (2) This script targets any POSIX shell, so it avoids extensions provided +# by Bash, Ksh, etc; in particular arrays are avoided. +# +# The "traditional" practice of packing multiple parameters into a +# space-separated string is a well documented source of bugs and security +# problems, so this is (mostly) avoided, by progressively accumulating +# options in "$@", and eventually passing that to Java. +# +# Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS, +# and GRADLE_OPTS) rely on word-splitting, this is performed explicitly; +# see the in-line comments for details. +# +# There are tweaks for specific operating systems such as AIX, CygWin, +# Darwin, MinGW, and NonStop. +# +# (3) This script is generated from the Groovy template +# https://github.com/gradle/gradle/blob/HEAD/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt +# within the Gradle project. +# +# You can find Gradle at https://github.com/gradle/gradle/. +# +############################################################################## + +# Attempt to set APP_HOME + +# Resolve links: $0 may be a link +app_path=$0 + +# Need this for daisy-chained symlinks. +while + APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path + [ -h "$app_path" ] +do + ls=$( ls -ld "$app_path" ) + link=${ls#*' -> '} + case $link in #( + /*) app_path=$link ;; #( + *) app_path=$APP_HOME$link ;; + esac +done + +# This is normally unused +# shellcheck disable=SC2034 +APP_BASE_NAME=${0##*/} +# Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036) +APP_HOME=$( cd "${APP_HOME:-./}" > /dev/null && pwd -P ) || exit + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD=maximum + +warn () { + echo "$*" +} >&2 + +die () { + echo + echo "$*" + echo + exit 1 +} >&2 + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +nonstop=false +case "$( uname )" in #( + CYGWIN* ) cygwin=true ;; #( + Darwin* ) darwin=true ;; #( + MSYS* | MINGW* ) msys=true ;; #( + NONSTOP* ) nonstop=true ;; +esac + +CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar + + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD=$JAVA_HOME/jre/sh/java + else + JAVACMD=$JAVA_HOME/bin/java + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD=java + if ! command -v java >/dev/null 2>&1 + then + die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +fi + +# Increase the maximum file descriptors if we can. +if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then + case $MAX_FD in #( + max*) + # In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC2039,SC3045 + MAX_FD=$( ulimit -H -n ) || + warn "Could not query maximum file descriptor limit" + esac + case $MAX_FD in #( + '' | soft) :;; #( + *) + # In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC2039,SC3045 + ulimit -n "$MAX_FD" || + warn "Could not set maximum file descriptor limit to $MAX_FD" + esac +fi + +# Collect all arguments for the java command, stacking in reverse order: +# * args from the command line +# * the main class name +# * -classpath +# * -D...appname settings +# * --module-path (only if needed) +# * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables. + +# For Cygwin or MSYS, switch paths to Windows format before running java +if "$cygwin" || "$msys" ; then + APP_HOME=$( cygpath --path --mixed "$APP_HOME" ) + CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" ) + + JAVACMD=$( cygpath --unix "$JAVACMD" ) + + # Now convert the arguments - kludge to limit ourselves to /bin/sh + for arg do + if + case $arg in #( + -*) false ;; # don't mess with options #( + /?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath + [ -e "$t" ] ;; #( + *) false ;; + esac + then + arg=$( cygpath --path --ignore --mixed "$arg" ) + fi + # Roll the args list around exactly as many times as the number of + # args, so each arg winds up back in the position where it started, but + # possibly modified. + # + # NB: a `for` loop captures its iteration list before it begins, so + # changing the positional parameters here affects neither the number of + # iterations, nor the values presented in `arg`. + shift # remove old arg + set -- "$@" "$arg" # push replacement arg + done +fi + + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m" "--enable-native-access=ALL-UNNAMED"' + +# Collect all arguments for the java command: +# * DEFAULT_JVM_OPTS, JAVA_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments, +# and any embedded shellness will be escaped. +# * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be +# treated as '${Hostname}' itself on the command line. + +set -- \ + "-Dorg.gradle.appname=$APP_BASE_NAME" \ + -classpath "$CLASSPATH" \ + org.gradle.wrapper.GradleWrapperMain \ + "$@" + +# Stop when "xargs" is not available. +if ! command -v xargs >/dev/null 2>&1 +then + die "xargs is not available" +fi + +# Use "xargs" to parse quoted args. +# +# With -n1 it outputs one arg per line, with the quotes and backslashes removed. +# +# In Bash we could simply go: +# +# readarray ARGS < <( xargs -n1 <<<"$var" ) && +# set -- "${ARGS[@]}" "$@" +# +# but POSIX shell has neither arrays nor command substitution, so instead we +# post-process each arg (as a line of input to sed) to backslash-escape any +# character that might be a shell metacharacter, then use eval to reverse +# that process (while maintaining the separation between arguments), and wrap +# the whole thing up as a single "set" statement. +# +# This will of course break if any of these variables contains a newline or +# an unmatched quote. +# + +eval "set -- $( + printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" | + xargs -n1 | + sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' | + tr '\n' ' ' + )" '"$@"' + +exec "$JAVACMD" "$@" diff --git a/apps/android/gradlew.bat b/apps/android/gradlew.bat new file mode 100644 index 0000000000000000000000000000000000000000..6f8e906658468ceadeadac1ed5a67da7e57b8483 --- /dev/null +++ b/apps/android/gradlew.bat @@ -0,0 +1,92 @@ +@rem +@rem Copyright 2015 the original author or authors. +@rem +@rem Licensed under the Apache License, Version 2.0 (the "License"); +@rem you may not use this file except in compliance with the License. +@rem You may obtain a copy of the License at +@rem +@rem https://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem + +@if "%DEBUG%"=="" @echo off +@rem ########################################################################## +@rem +@rem Gradle startup script for Windows +@rem +@rem ########################################################################## + +@rem Set local scope for the variables with windows NT shell +if "%OS%"=="Windows_NT" setlocal + +set DIRNAME=%~dp0 +if "%DIRNAME%"=="" set DIRNAME=. +@rem This is normally unused +set APP_BASE_NAME=%~n0 +set APP_HOME=%DIRNAME% + +@rem Resolve any "." and ".." in APP_HOME to make it shorter. +for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi + +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" "--enable-native-access=ALL-UNNAMED" + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if %ERRORLEVEL% equ 0 goto execute + +echo. 1>&2 +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 1>&2 +echo. 1>&2 +echo Please set the JAVA_HOME variable in your environment to match the 1>&2 +echo location of your Java installation. 1>&2 + +goto fail + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto execute + +echo. 1>&2 +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 1>&2 +echo. 1>&2 +echo Please set the JAVA_HOME variable in your environment to match the 1>&2 +echo location of your Java installation. 1>&2 + +goto fail + +:execute +@rem Setup the command line + +set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar + + +@rem Execute Gradle +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* + +:end +@rem End local scope for the variables with windows NT shell +if %ERRORLEVEL% equ 0 goto mainEnd + +:fail +rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of +rem the _cmd.exe /c_ return code! +set EXIT_CODE=%ERRORLEVEL% +if %EXIT_CODE% equ 0 set EXIT_CODE=1 +if not ""=="%GRADLE_EXIT_CONSOLE%" exit %EXIT_CODE% +exit /b %EXIT_CODE% + +:mainEnd +if "%OS%"=="Windows_NT" endlocal + +:omega diff --git a/apps/android/settings.gradle.kts b/apps/android/settings.gradle.kts new file mode 100644 index 0000000000000000000000000000000000000000..b3b43a4455016b567af2e5076d3c62235e4cd7f8 --- /dev/null +++ b/apps/android/settings.gradle.kts @@ -0,0 +1,18 @@ +pluginManagement { + repositories { + google() + mavenCentral() + gradlePluginPortal() + } +} + +dependencyResolutionManagement { + repositoriesMode.set(RepositoriesMode.FAIL_ON_PROJECT_REPOS) + repositories { + google() + mavenCentral() + } +} + +rootProject.name = "OpenClawNodeAndroid" +include(":app") diff --git a/apps/ios/.swiftlint.yml b/apps/ios/.swiftlint.yml new file mode 100644 index 0000000000000000000000000000000000000000..fc8509c83859e5f87ed49fa2633462c7cc53832e --- /dev/null +++ b/apps/ios/.swiftlint.yml @@ -0,0 +1,5 @@ +parent_config: ../../.swiftlint.yml + +included: + - Sources + - ../shared/ClawdisNodeKit/Sources diff --git a/apps/ios/README.md b/apps/ios/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7af4d5d5da6b425db29dc4a0d6ad800a351b906a --- /dev/null +++ b/apps/ios/README.md @@ -0,0 +1,28 @@ +# OpenClaw (iOS) + +Internal-only SwiftUI app scaffold. + +## Lint/format (required) +```bash +brew install swiftformat swiftlint +``` + +## Generate the Xcode project +```bash +cd apps/ios +xcodegen generate +open OpenClaw.xcodeproj +``` + +## Shared packages +- `../shared/OpenClawKit` — shared types/constants used by iOS (and later macOS bridge + gateway routing). + +## fastlane +```bash +brew install fastlane + +cd apps/ios +fastlane lanes +``` + +See `apps/ios/fastlane/SETUP.md` for App Store Connect auth + upload lanes. diff --git a/apps/ios/Sources/Assets.xcassets/AppIcon.appiconset/Contents.json b/apps/ios/Sources/Assets.xcassets/AppIcon.appiconset/Contents.json new file mode 100644 index 0000000000000000000000000000000000000000..13847b5b5bf74e7c826ca4d1bd697521feeb2e3b --- /dev/null +++ b/apps/ios/Sources/Assets.xcassets/AppIcon.appiconset/Contents.json @@ -0,0 +1,31 @@ +{ + "images" : [ + { "filename" : "icon-20@1x.png", "idiom" : "ipad", "scale" : "1x", "size" : "20x20" }, + { "filename" : "icon-20@2x.png", "idiom" : "ipad", "scale" : "2x", "size" : "20x20" }, + { "filename" : "icon-20@2x.png", "idiom" : "iphone","scale" : "2x", "size" : "20x20" }, + { "filename" : "icon-20@3x.png", "idiom" : "iphone","scale" : "3x", "size" : "20x20" }, + + { "filename" : "icon-29@1x.png", "idiom" : "ipad", "scale" : "1x", "size" : "29x29" }, + { "filename" : "icon-29@2x.png", "idiom" : "ipad", "scale" : "2x", "size" : "29x29" }, + { "filename" : "icon-29@2x.png", "idiom" : "iphone","scale" : "2x", "size" : "29x29" }, + { "filename" : "icon-29@3x.png", "idiom" : "iphone","scale" : "3x", "size" : "29x29" }, + + { "filename" : "icon-40@1x.png", "idiom" : "ipad", "scale" : "1x", "size" : "40x40" }, + { "filename" : "icon-40@2x.png", "idiom" : "ipad", "scale" : "2x", "size" : "40x40" }, + { "filename" : "icon-40@2x.png", "idiom" : "iphone","scale" : "2x", "size" : "40x40" }, + { "filename" : "icon-40@3x.png", "idiom" : "iphone","scale" : "3x", "size" : "40x40" }, + + { "filename" : "icon-60@2x.png", "idiom" : "iphone","scale" : "2x", "size" : "60x60" }, + { "filename" : "icon-60@3x.png", "idiom" : "iphone","scale" : "3x", "size" : "60x60" }, + + { "filename" : "icon-76@2x.png", "idiom" : "ipad", "scale" : "2x", "size" : "76x76" }, + + { "filename" : "icon-83.5@2x.png", "idiom" : "ipad", "scale" : "2x", "size" : "83.5x83.5" }, + + { "filename" : "icon-1024.png", "idiom" : "ios-marketing", "scale" : "1x", "size" : "1024x1024" } + ], + "info" : { + "author" : "xcode", + "version" : 1 + } +} diff --git a/apps/ios/Sources/Assets.xcassets/AppIcon.appiconset/icon-1024.png b/apps/ios/Sources/Assets.xcassets/AppIcon.appiconset/icon-1024.png new file mode 100644 index 0000000000000000000000000000000000000000..e79a671cc8710b2e64e65b3518c492db224abc0e --- /dev/null +++ b/apps/ios/Sources/Assets.xcassets/AppIcon.appiconset/icon-1024.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3da94d4f6d5e6e3d4ec2d92becf960f1d8090ecd67a5674cc6b9794ba3dbb2ef +size 1393705 diff --git a/apps/ios/Sources/Assets.xcassets/AppIcon.appiconset/icon-20@1x.png b/apps/ios/Sources/Assets.xcassets/AppIcon.appiconset/icon-20@1x.png new file mode 100644 index 0000000000000000000000000000000000000000..0aa1506a095c4c7127fb0aac580df412ea745865 Binary files /dev/null and b/apps/ios/Sources/Assets.xcassets/AppIcon.appiconset/icon-20@1x.png differ diff --git a/apps/ios/Sources/Assets.xcassets/AppIcon.appiconset/icon-20@2x.png b/apps/ios/Sources/Assets.xcassets/AppIcon.appiconset/icon-20@2x.png new file mode 100644 index 0000000000000000000000000000000000000000..dd8a14724eb439e481ecbc7009f76ee6cd9d2151 Binary files /dev/null and b/apps/ios/Sources/Assets.xcassets/AppIcon.appiconset/icon-20@2x.png differ diff --git a/apps/ios/Sources/Assets.xcassets/AppIcon.appiconset/icon-20@3x.png b/apps/ios/Sources/Assets.xcassets/AppIcon.appiconset/icon-20@3x.png new file mode 100644 index 0000000000000000000000000000000000000000..ca160dc2e84624c09157b46647ebfb9e0b963a08 Binary files /dev/null and b/apps/ios/Sources/Assets.xcassets/AppIcon.appiconset/icon-20@3x.png differ diff --git a/apps/ios/Sources/Assets.xcassets/AppIcon.appiconset/icon-29@1x.png b/apps/ios/Sources/Assets.xcassets/AppIcon.appiconset/icon-29@1x.png new file mode 100644 index 0000000000000000000000000000000000000000..9020a8672d3b79b3a802ce37094b2eaa8ebb275b Binary files /dev/null and b/apps/ios/Sources/Assets.xcassets/AppIcon.appiconset/icon-29@1x.png differ diff --git a/apps/ios/Sources/Assets.xcassets/AppIcon.appiconset/icon-29@2x.png b/apps/ios/Sources/Assets.xcassets/AppIcon.appiconset/icon-29@2x.png new file mode 100644 index 0000000000000000000000000000000000000000..ff85b417fece435cf344d2483afd3200325f3274 Binary files /dev/null and b/apps/ios/Sources/Assets.xcassets/AppIcon.appiconset/icon-29@2x.png differ diff --git a/apps/ios/Sources/Assets.xcassets/AppIcon.appiconset/icon-29@3x.png b/apps/ios/Sources/Assets.xcassets/AppIcon.appiconset/icon-29@3x.png new file mode 100644 index 0000000000000000000000000000000000000000..e12fff03140063443d59f7db08bff3ad2d91217b Binary files /dev/null and b/apps/ios/Sources/Assets.xcassets/AppIcon.appiconset/icon-29@3x.png differ diff --git a/apps/ios/Sources/Assets.xcassets/AppIcon.appiconset/icon-40@1x.png b/apps/ios/Sources/Assets.xcassets/AppIcon.appiconset/icon-40@1x.png new file mode 100644 index 0000000000000000000000000000000000000000..dd8a14724eb439e481ecbc7009f76ee6cd9d2151 Binary files /dev/null and b/apps/ios/Sources/Assets.xcassets/AppIcon.appiconset/icon-40@1x.png differ diff --git a/apps/ios/Sources/Assets.xcassets/AppIcon.appiconset/icon-40@2x.png b/apps/ios/Sources/Assets.xcassets/AppIcon.appiconset/icon-40@2x.png new file mode 100644 index 0000000000000000000000000000000000000000..9b3da5155efec127adbe4df5bf61d57ba5e53666 Binary files /dev/null and b/apps/ios/Sources/Assets.xcassets/AppIcon.appiconset/icon-40@2x.png differ diff --git a/apps/ios/Sources/Assets.xcassets/AppIcon.appiconset/icon-40@3x.png b/apps/ios/Sources/Assets.xcassets/AppIcon.appiconset/icon-40@3x.png new file mode 100644 index 0000000000000000000000000000000000000000..f57a0c1323c68de3cf69ad1eb96ec436709eec1e Binary files /dev/null and b/apps/ios/Sources/Assets.xcassets/AppIcon.appiconset/icon-40@3x.png differ diff --git a/apps/ios/Sources/Assets.xcassets/AppIcon.appiconset/icon-60@2x.png b/apps/ios/Sources/Assets.xcassets/AppIcon.appiconset/icon-60@2x.png new file mode 100644 index 0000000000000000000000000000000000000000..f57a0c1323c68de3cf69ad1eb96ec436709eec1e Binary files /dev/null and b/apps/ios/Sources/Assets.xcassets/AppIcon.appiconset/icon-60@2x.png differ diff --git a/apps/ios/Sources/Assets.xcassets/AppIcon.appiconset/icon-60@3x.png b/apps/ios/Sources/Assets.xcassets/AppIcon.appiconset/icon-60@3x.png new file mode 100644 index 0000000000000000000000000000000000000000..b94278f29d07b9b42b2fb0bf889c192ecb14d1e6 Binary files /dev/null and b/apps/ios/Sources/Assets.xcassets/AppIcon.appiconset/icon-60@3x.png differ diff --git a/apps/ios/Sources/Assets.xcassets/AppIcon.appiconset/icon-76@2x.png b/apps/ios/Sources/Assets.xcassets/AppIcon.appiconset/icon-76@2x.png new file mode 100644 index 0000000000000000000000000000000000000000..2d6240dc679c4d71f262e0196bc8ea275f4f4209 Binary files /dev/null and b/apps/ios/Sources/Assets.xcassets/AppIcon.appiconset/icon-76@2x.png differ diff --git a/apps/ios/Sources/Assets.xcassets/AppIcon.appiconset/icon-83.5@2x.png b/apps/ios/Sources/Assets.xcassets/AppIcon.appiconset/icon-83.5@2x.png new file mode 100644 index 0000000000000000000000000000000000000000..7321091c561e01119c292785b08f7889397d02f1 Binary files /dev/null and b/apps/ios/Sources/Assets.xcassets/AppIcon.appiconset/icon-83.5@2x.png differ diff --git a/apps/ios/Sources/Assets.xcassets/Contents.json b/apps/ios/Sources/Assets.xcassets/Contents.json new file mode 100644 index 0000000000000000000000000000000000000000..73c00596a7fca3f3d4bdd64053b69d86745f9e10 --- /dev/null +++ b/apps/ios/Sources/Assets.xcassets/Contents.json @@ -0,0 +1,6 @@ +{ + "info" : { + "author" : "xcode", + "version" : 1 + } +} diff --git a/apps/ios/Sources/Camera/CameraController.swift b/apps/ios/Sources/Camera/CameraController.swift new file mode 100644 index 0000000000000000000000000000000000000000..e76dbeeabb90efedd7a994594af5522a58b84a03 --- /dev/null +++ b/apps/ios/Sources/Camera/CameraController.swift @@ -0,0 +1,406 @@ +import AVFoundation +import OpenClawKit +import Foundation + +actor CameraController { + struct CameraDeviceInfo: Codable, Sendable { + var id: String + var name: String + var position: String + var deviceType: String + } + + enum CameraError: LocalizedError, Sendable { + case cameraUnavailable + case microphoneUnavailable + case permissionDenied(kind: String) + case invalidParams(String) + case captureFailed(String) + case exportFailed(String) + + var errorDescription: String? { + switch self { + case .cameraUnavailable: + "Camera unavailable" + case .microphoneUnavailable: + "Microphone unavailable" + case let .permissionDenied(kind): + "\(kind) permission denied" + case let .invalidParams(msg): + msg + case let .captureFailed(msg): + msg + case let .exportFailed(msg): + msg + } + } + } + + func snap(params: OpenClawCameraSnapParams) async throws -> ( + format: String, + base64: String, + width: Int, + height: Int) + { + let facing = params.facing ?? .front + let format = params.format ?? .jpg + // Default to a reasonable max width to keep gateway payload sizes manageable. + // If you need the full-res photo, explicitly request a larger maxWidth. + let maxWidth = params.maxWidth.flatMap { $0 > 0 ? $0 : nil } ?? 1600 + let quality = Self.clampQuality(params.quality) + let delayMs = max(0, params.delayMs ?? 0) + + try await self.ensureAccess(for: .video) + + let session = AVCaptureSession() + session.sessionPreset = .photo + + guard let device = Self.pickCamera(facing: facing, deviceId: params.deviceId) else { + throw CameraError.cameraUnavailable + } + + let input = try AVCaptureDeviceInput(device: device) + guard session.canAddInput(input) else { + throw CameraError.captureFailed("Failed to add camera input") + } + session.addInput(input) + + let output = AVCapturePhotoOutput() + guard session.canAddOutput(output) else { + throw CameraError.captureFailed("Failed to add photo output") + } + session.addOutput(output) + output.maxPhotoQualityPrioritization = .quality + + session.startRunning() + defer { session.stopRunning() } + await Self.warmUpCaptureSession() + await Self.sleepDelayMs(delayMs) + + let settings: AVCapturePhotoSettings = { + if output.availablePhotoCodecTypes.contains(.jpeg) { + return AVCapturePhotoSettings(format: [AVVideoCodecKey: AVVideoCodecType.jpeg]) + } + return AVCapturePhotoSettings() + }() + settings.photoQualityPrioritization = .quality + + var delegate: PhotoCaptureDelegate? + let rawData: Data = try await withCheckedThrowingContinuation { cont in + let d = PhotoCaptureDelegate(cont) + delegate = d + output.capturePhoto(with: settings, delegate: d) + } + withExtendedLifetime(delegate) {} + + let maxPayloadBytes = 5 * 1024 * 1024 + // Base64 inflates payloads by ~4/3; cap encoded bytes so the payload stays under 5MB (API limit). + let maxEncodedBytes = (maxPayloadBytes / 4) * 3 + let res = try JPEGTranscoder.transcodeToJPEG( + imageData: rawData, + maxWidthPx: maxWidth, + quality: quality, + maxBytes: maxEncodedBytes) + + return ( + format: format.rawValue, + base64: res.data.base64EncodedString(), + width: res.widthPx, + height: res.heightPx) + } + + func clip(params: OpenClawCameraClipParams) async throws -> ( + format: String, + base64: String, + durationMs: Int, + hasAudio: Bool) + { + let facing = params.facing ?? .front + let durationMs = Self.clampDurationMs(params.durationMs) + let includeAudio = params.includeAudio ?? true + let format = params.format ?? .mp4 + + try await self.ensureAccess(for: .video) + if includeAudio { + try await self.ensureAccess(for: .audio) + } + + let session = AVCaptureSession() + session.sessionPreset = .high + + guard let camera = Self.pickCamera(facing: facing, deviceId: params.deviceId) else { + throw CameraError.cameraUnavailable + } + let cameraInput = try AVCaptureDeviceInput(device: camera) + guard session.canAddInput(cameraInput) else { + throw CameraError.captureFailed("Failed to add camera input") + } + session.addInput(cameraInput) + + if includeAudio { + guard let mic = AVCaptureDevice.default(for: .audio) else { + throw CameraError.microphoneUnavailable + } + let micInput = try AVCaptureDeviceInput(device: mic) + if session.canAddInput(micInput) { + session.addInput(micInput) + } else { + throw CameraError.captureFailed("Failed to add microphone input") + } + } + + let output = AVCaptureMovieFileOutput() + guard session.canAddOutput(output) else { + throw CameraError.captureFailed("Failed to add movie output") + } + session.addOutput(output) + output.maxRecordedDuration = CMTime(value: Int64(durationMs), timescale: 1000) + + session.startRunning() + defer { session.stopRunning() } + await Self.warmUpCaptureSession() + + let movURL = FileManager().temporaryDirectory + .appendingPathComponent("openclaw-camera-\(UUID().uuidString).mov") + let mp4URL = FileManager().temporaryDirectory + .appendingPathComponent("openclaw-camera-\(UUID().uuidString).mp4") + + defer { + try? FileManager().removeItem(at: movURL) + try? FileManager().removeItem(at: mp4URL) + } + + var delegate: MovieFileDelegate? + let recordedURL: URL = try await withCheckedThrowingContinuation { cont in + let d = MovieFileDelegate(cont) + delegate = d + output.startRecording(to: movURL, recordingDelegate: d) + } + withExtendedLifetime(delegate) {} + + // Transcode .mov -> .mp4 for easier downstream handling. + try await Self.exportToMP4(inputURL: recordedURL, outputURL: mp4URL) + + let data = try Data(contentsOf: mp4URL) + return ( + format: format.rawValue, + base64: data.base64EncodedString(), + durationMs: durationMs, + hasAudio: includeAudio) + } + + func listDevices() -> [CameraDeviceInfo] { + return Self.discoverVideoDevices().map { device in + CameraDeviceInfo( + id: device.uniqueID, + name: device.localizedName, + position: Self.positionLabel(device.position), + deviceType: device.deviceType.rawValue) + } + } + + private func ensureAccess(for mediaType: AVMediaType) async throws { + let status = AVCaptureDevice.authorizationStatus(for: mediaType) + switch status { + case .authorized: + return + case .notDetermined: + let ok = await withCheckedContinuation(isolation: nil) { cont in + AVCaptureDevice.requestAccess(for: mediaType) { granted in + cont.resume(returning: granted) + } + } + if !ok { + throw CameraError.permissionDenied(kind: mediaType == .video ? "Camera" : "Microphone") + } + case .denied, .restricted: + throw CameraError.permissionDenied(kind: mediaType == .video ? "Camera" : "Microphone") + @unknown default: + throw CameraError.permissionDenied(kind: mediaType == .video ? "Camera" : "Microphone") + } + } + + private nonisolated static func pickCamera( + facing: OpenClawCameraFacing, + deviceId: String?) -> AVCaptureDevice? + { + if let deviceId, !deviceId.isEmpty { + if let match = Self.discoverVideoDevices().first(where: { $0.uniqueID == deviceId }) { + return match + } + } + let position: AVCaptureDevice.Position = (facing == .front) ? .front : .back + if let device = AVCaptureDevice.default(.builtInWideAngleCamera, for: .video, position: position) { + return device + } + // Fall back to any default camera (e.g. simulator / unusual device configurations). + return AVCaptureDevice.default(for: .video) + } + + private nonisolated static func positionLabel(_ position: AVCaptureDevice.Position) -> String { + switch position { + case .front: "front" + case .back: "back" + default: "unspecified" + } + } + + private nonisolated static func discoverVideoDevices() -> [AVCaptureDevice] { + let types: [AVCaptureDevice.DeviceType] = [ + .builtInWideAngleCamera, + .builtInUltraWideCamera, + .builtInTelephotoCamera, + .builtInDualCamera, + .builtInDualWideCamera, + .builtInTripleCamera, + .builtInTrueDepthCamera, + .builtInLiDARDepthCamera, + ] + let session = AVCaptureDevice.DiscoverySession( + deviceTypes: types, + mediaType: .video, + position: .unspecified) + return session.devices + } + + nonisolated static func clampQuality(_ quality: Double?) -> Double { + let q = quality ?? 0.9 + return min(1.0, max(0.05, q)) + } + + nonisolated static func clampDurationMs(_ ms: Int?) -> Int { + let v = ms ?? 3000 + // Keep clips short by default; avoid huge base64 payloads on the gateway. + return min(60000, max(250, v)) + } + + private nonisolated static func exportToMP4(inputURL: URL, outputURL: URL) async throws { + let asset = AVURLAsset(url: inputURL) + guard let exporter = AVAssetExportSession(asset: asset, presetName: AVAssetExportPresetMediumQuality) else { + throw CameraError.exportFailed("Failed to create export session") + } + exporter.shouldOptimizeForNetworkUse = true + + if #available(iOS 18.0, tvOS 18.0, visionOS 2.0, *) { + do { + try await exporter.export(to: outputURL, as: .mp4) + return + } catch { + throw CameraError.exportFailed(error.localizedDescription) + } + } else { + exporter.outputURL = outputURL + exporter.outputFileType = .mp4 + + try await withCheckedThrowingContinuation(isolation: nil) { (cont: CheckedContinuation) in + exporter.exportAsynchronously { + cont.resume(returning: ()) + } + } + + switch exporter.status { + case .completed: + return + case .failed: + throw CameraError.exportFailed(exporter.error?.localizedDescription ?? "export failed") + case .cancelled: + throw CameraError.exportFailed("export cancelled") + default: + throw CameraError.exportFailed("export did not complete") + } + } + } + + private nonisolated static func warmUpCaptureSession() async { + // A short delay after `startRunning()` significantly reduces "blank first frame" captures on some devices. + try? await Task.sleep(nanoseconds: 150_000_000) // 150ms + } + + private nonisolated static func sleepDelayMs(_ delayMs: Int) async { + guard delayMs > 0 else { return } + let maxDelayMs = 10 * 1000 + let ns = UInt64(min(delayMs, maxDelayMs)) * UInt64(NSEC_PER_MSEC) + try? await Task.sleep(nanoseconds: ns) + } +} + +private final class PhotoCaptureDelegate: NSObject, AVCapturePhotoCaptureDelegate { + private let continuation: CheckedContinuation + private var didResume = false + + init(_ continuation: CheckedContinuation) { + self.continuation = continuation + } + + func photoOutput( + _ output: AVCapturePhotoOutput, + didFinishProcessingPhoto photo: AVCapturePhoto, + error: Error?) + { + guard !self.didResume else { return } + self.didResume = true + + if let error { + self.continuation.resume(throwing: error) + return + } + guard let data = photo.fileDataRepresentation() else { + self.continuation.resume( + throwing: NSError(domain: "Camera", code: 1, userInfo: [ + NSLocalizedDescriptionKey: "photo data missing", + ])) + return + } + if data.isEmpty { + self.continuation.resume( + throwing: NSError(domain: "Camera", code: 2, userInfo: [ + NSLocalizedDescriptionKey: "photo data empty", + ])) + return + } + self.continuation.resume(returning: data) + } + + func photoOutput( + _ output: AVCapturePhotoOutput, + didFinishCaptureFor resolvedSettings: AVCaptureResolvedPhotoSettings, + error: Error?) + { + guard let error else { return } + guard !self.didResume else { return } + self.didResume = true + self.continuation.resume(throwing: error) + } +} + +private final class MovieFileDelegate: NSObject, AVCaptureFileOutputRecordingDelegate { + private let continuation: CheckedContinuation + private var didResume = false + + init(_ continuation: CheckedContinuation) { + self.continuation = continuation + } + + func fileOutput( + _ output: AVCaptureFileOutput, + didFinishRecordingTo outputFileURL: URL, + from connections: [AVCaptureConnection], + error: Error?) + { + guard !self.didResume else { return } + self.didResume = true + + if let error { + let ns = error as NSError + if ns.domain == AVFoundationErrorDomain, + ns.code == AVError.maximumDurationReached.rawValue + { + self.continuation.resume(returning: outputFileURL) + return + } + self.continuation.resume(throwing: error) + return + } + self.continuation.resume(returning: outputFileURL) + } +} diff --git a/apps/ios/Sources/Chat/ChatSheet.swift b/apps/ios/Sources/Chat/ChatSheet.swift new file mode 100644 index 0000000000000000000000000000000000000000..6b8fffd23d0e7250d641ccabdbfe0e1d8b4d41ba --- /dev/null +++ b/apps/ios/Sources/Chat/ChatSheet.swift @@ -0,0 +1,39 @@ +import OpenClawChatUI +import OpenClawKit +import SwiftUI + +struct ChatSheet: View { + @Environment(\.dismiss) private var dismiss + @State private var viewModel: OpenClawChatViewModel + private let userAccent: Color? + + init(gateway: GatewayNodeSession, sessionKey: String, userAccent: Color? = nil) { + let transport = IOSGatewayChatTransport(gateway: gateway) + self._viewModel = State( + initialValue: OpenClawChatViewModel( + sessionKey: sessionKey, + transport: transport)) + self.userAccent = userAccent + } + + var body: some View { + NavigationStack { + OpenClawChatView( + viewModel: self.viewModel, + showsSessionSwitcher: true, + userAccent: self.userAccent) + .navigationTitle("Chat") + .navigationBarTitleDisplayMode(.inline) + .toolbar { + ToolbarItem(placement: .topBarTrailing) { + Button { + self.dismiss() + } label: { + Image(systemName: "xmark") + } + .accessibilityLabel("Close") + } + } + } + } +} diff --git a/apps/ios/Sources/Chat/IOSGatewayChatTransport.swift b/apps/ios/Sources/Chat/IOSGatewayChatTransport.swift new file mode 100644 index 0000000000000000000000000000000000000000..3c828551ada0b9cb265e22e64498c9b2f590333e --- /dev/null +++ b/apps/ios/Sources/Chat/IOSGatewayChatTransport.swift @@ -0,0 +1,129 @@ +import OpenClawChatUI +import OpenClawKit +import OpenClawProtocol +import Foundation + +struct IOSGatewayChatTransport: OpenClawChatTransport, Sendable { + private let gateway: GatewayNodeSession + + init(gateway: GatewayNodeSession) { + self.gateway = gateway + } + + func abortRun(sessionKey: String, runId: String) async throws { + struct Params: Codable { + var sessionKey: String + var runId: String + } + let data = try JSONEncoder().encode(Params(sessionKey: sessionKey, runId: runId)) + let json = String(data: data, encoding: .utf8) + _ = try await self.gateway.request(method: "chat.abort", paramsJSON: json, timeoutSeconds: 10) + } + + func listSessions(limit: Int?) async throws -> OpenClawChatSessionsListResponse { + struct Params: Codable { + var includeGlobal: Bool + var includeUnknown: Bool + var limit: Int? + } + let data = try JSONEncoder().encode(Params(includeGlobal: true, includeUnknown: false, limit: limit)) + let json = String(data: data, encoding: .utf8) + let res = try await self.gateway.request(method: "sessions.list", paramsJSON: json, timeoutSeconds: 15) + return try JSONDecoder().decode(OpenClawChatSessionsListResponse.self, from: res) + } + + func setActiveSessionKey(_ sessionKey: String) async throws { + struct Subscribe: Codable { var sessionKey: String } + let data = try JSONEncoder().encode(Subscribe(sessionKey: sessionKey)) + let json = String(data: data, encoding: .utf8) + await self.gateway.sendEvent(event: "chat.subscribe", payloadJSON: json) + } + + func requestHistory(sessionKey: String) async throws -> OpenClawChatHistoryPayload { + struct Params: Codable { var sessionKey: String } + let data = try JSONEncoder().encode(Params(sessionKey: sessionKey)) + let json = String(data: data, encoding: .utf8) + let res = try await self.gateway.request(method: "chat.history", paramsJSON: json, timeoutSeconds: 15) + return try JSONDecoder().decode(OpenClawChatHistoryPayload.self, from: res) + } + + func sendMessage( + sessionKey: String, + message: String, + thinking: String, + idempotencyKey: String, + attachments: [OpenClawChatAttachmentPayload]) async throws -> OpenClawChatSendResponse + { + struct Params: Codable { + var sessionKey: String + var message: String + var thinking: String + var attachments: [OpenClawChatAttachmentPayload]? + var timeoutMs: Int + var idempotencyKey: String + } + + let params = Params( + sessionKey: sessionKey, + message: message, + thinking: thinking, + attachments: attachments.isEmpty ? nil : attachments, + timeoutMs: 30000, + idempotencyKey: idempotencyKey) + let data = try JSONEncoder().encode(params) + let json = String(data: data, encoding: .utf8) + let res = try await self.gateway.request(method: "chat.send", paramsJSON: json, timeoutSeconds: 35) + return try JSONDecoder().decode(OpenClawChatSendResponse.self, from: res) + } + + func requestHealth(timeoutMs: Int) async throws -> Bool { + let seconds = max(1, Int(ceil(Double(timeoutMs) / 1000.0))) + let res = try await self.gateway.request(method: "health", paramsJSON: nil, timeoutSeconds: seconds) + return (try? JSONDecoder().decode(OpenClawGatewayHealthOK.self, from: res))?.ok ?? true + } + + func events() -> AsyncStream { + AsyncStream { continuation in + let task = Task { + let stream = await self.gateway.subscribeServerEvents() + for await evt in stream { + if Task.isCancelled { return } + switch evt.event { + case "tick": + continuation.yield(.tick) + case "seqGap": + continuation.yield(.seqGap) + case "health": + guard let payload = evt.payload else { break } + let ok = (try? GatewayPayloadDecoding.decode( + payload, + as: OpenClawGatewayHealthOK.self))?.ok ?? true + continuation.yield(.health(ok: ok)) + case "chat": + guard let payload = evt.payload else { break } + if let chatPayload = try? GatewayPayloadDecoding.decode( + payload, + as: OpenClawChatEventPayload.self) + { + continuation.yield(.chat(chatPayload)) + } + case "agent": + guard let payload = evt.payload else { break } + if let agentPayload = try? GatewayPayloadDecoding.decode( + payload, + as: OpenClawAgentEventPayload.self) + { + continuation.yield(.agent(agentPayload)) + } + default: + break + } + } + } + + continuation.onTermination = { @Sendable _ in + task.cancel() + } + } + } +} diff --git a/apps/ios/Sources/Gateway/GatewayConnectionController.swift b/apps/ios/Sources/Gateway/GatewayConnectionController.swift new file mode 100644 index 0000000000000000000000000000000000000000..65d099c01064ff336f6e93a362c744778d2447dd --- /dev/null +++ b/apps/ios/Sources/Gateway/GatewayConnectionController.swift @@ -0,0 +1,434 @@ +import OpenClawKit +import Darwin +import Foundation +import Network +import Observation +import SwiftUI +import UIKit + +@MainActor +@Observable +final class GatewayConnectionController { + private(set) var gateways: [GatewayDiscoveryModel.DiscoveredGateway] = [] + private(set) var discoveryStatusText: String = "Idle" + private(set) var discoveryDebugLog: [GatewayDiscoveryModel.DebugLogEntry] = [] + + private let discovery = GatewayDiscoveryModel() + private weak var appModel: NodeAppModel? + private var didAutoConnect = false + + init(appModel: NodeAppModel, startDiscovery: Bool = true) { + self.appModel = appModel + + GatewaySettingsStore.bootstrapPersistence() + let defaults = UserDefaults.standard + self.discovery.setDebugLoggingEnabled(defaults.bool(forKey: "gateway.discovery.debugLogs")) + + self.updateFromDiscovery() + self.observeDiscovery() + + if startDiscovery { + self.discovery.start() + } + } + + func setDiscoveryDebugLoggingEnabled(_ enabled: Bool) { + self.discovery.setDebugLoggingEnabled(enabled) + } + + func setScenePhase(_ phase: ScenePhase) { + switch phase { + case .background: + self.discovery.stop() + case .active, .inactive: + self.discovery.start() + @unknown default: + self.discovery.start() + } + } + + func connect(_ gateway: GatewayDiscoveryModel.DiscoveredGateway) async { + let instanceId = UserDefaults.standard.string(forKey: "node.instanceId")? + .trimmingCharacters(in: .whitespacesAndNewlines) ?? "" + let token = GatewaySettingsStore.loadGatewayToken(instanceId: instanceId) + let password = GatewaySettingsStore.loadGatewayPassword(instanceId: instanceId) + guard let host = self.resolveGatewayHost(gateway) else { return } + let port = gateway.gatewayPort ?? 18789 + let tlsParams = self.resolveDiscoveredTLSParams(gateway: gateway) + guard let url = self.buildGatewayURL( + host: host, + port: port, + useTLS: tlsParams?.required == true) + else { return } + self.didAutoConnect = true + self.startAutoConnect( + url: url, + gatewayStableID: gateway.stableID, + tls: tlsParams, + token: token, + password: password) + } + + func connectManual(host: String, port: Int, useTLS: Bool) async { + let instanceId = UserDefaults.standard.string(forKey: "node.instanceId")? + .trimmingCharacters(in: .whitespacesAndNewlines) ?? "" + let token = GatewaySettingsStore.loadGatewayToken(instanceId: instanceId) + let password = GatewaySettingsStore.loadGatewayPassword(instanceId: instanceId) + let stableID = self.manualStableID(host: host, port: port) + let tlsParams = self.resolveManualTLSParams(stableID: stableID, tlsEnabled: useTLS) + guard let url = self.buildGatewayURL( + host: host, + port: port, + useTLS: tlsParams?.required == true) + else { return } + self.didAutoConnect = true + self.startAutoConnect( + url: url, + gatewayStableID: stableID, + tls: tlsParams, + token: token, + password: password) + } + + private func updateFromDiscovery() { + let newGateways = self.discovery.gateways + self.gateways = newGateways + self.discoveryStatusText = self.discovery.statusText + self.discoveryDebugLog = self.discovery.debugLog + self.updateLastDiscoveredGateway(from: newGateways) + self.maybeAutoConnect() + } + + private func observeDiscovery() { + withObservationTracking { + _ = self.discovery.gateways + _ = self.discovery.statusText + _ = self.discovery.debugLog + } onChange: { [weak self] in + Task { @MainActor in + guard let self else { return } + self.updateFromDiscovery() + self.observeDiscovery() + } + } + } + + private func maybeAutoConnect() { + guard !self.didAutoConnect else { return } + guard let appModel = self.appModel else { return } + guard appModel.gatewayServerName == nil else { return } + + let defaults = UserDefaults.standard + let manualEnabled = defaults.bool(forKey: "gateway.manual.enabled") + + let instanceId = defaults.string(forKey: "node.instanceId")? + .trimmingCharacters(in: .whitespacesAndNewlines) ?? "" + guard !instanceId.isEmpty else { return } + + let token = GatewaySettingsStore.loadGatewayToken(instanceId: instanceId) + let password = GatewaySettingsStore.loadGatewayPassword(instanceId: instanceId) + + if manualEnabled { + let manualHost = defaults.string(forKey: "gateway.manual.host")? + .trimmingCharacters(in: .whitespacesAndNewlines) ?? "" + guard !manualHost.isEmpty else { return } + + let manualPort = defaults.integer(forKey: "gateway.manual.port") + let resolvedPort = manualPort > 0 ? manualPort : 18789 + let manualTLS = defaults.bool(forKey: "gateway.manual.tls") + + let stableID = self.manualStableID(host: manualHost, port: resolvedPort) + let tlsParams = self.resolveManualTLSParams(stableID: stableID, tlsEnabled: manualTLS) + + guard let url = self.buildGatewayURL( + host: manualHost, + port: resolvedPort, + useTLS: tlsParams?.required == true) + else { return } + + self.didAutoConnect = true + self.startAutoConnect( + url: url, + gatewayStableID: stableID, + tls: tlsParams, + token: token, + password: password) + return + } + + let preferredStableID = defaults.string(forKey: "gateway.preferredStableID")? + .trimmingCharacters(in: .whitespacesAndNewlines) ?? "" + let lastDiscoveredStableID = defaults.string(forKey: "gateway.lastDiscoveredStableID")? + .trimmingCharacters(in: .whitespacesAndNewlines) ?? "" + + let candidates = [preferredStableID, lastDiscoveredStableID].filter { !$0.isEmpty } + guard let targetStableID = candidates.first(where: { id in + self.gateways.contains(where: { $0.stableID == id }) + }) else { return } + + guard let target = self.gateways.first(where: { $0.stableID == targetStableID }) else { return } + guard let host = self.resolveGatewayHost(target) else { return } + let port = target.gatewayPort ?? 18789 + let tlsParams = self.resolveDiscoveredTLSParams(gateway: target) + guard let url = self.buildGatewayURL(host: host, port: port, useTLS: tlsParams?.required == true) + else { return } + + self.didAutoConnect = true + self.startAutoConnect( + url: url, + gatewayStableID: target.stableID, + tls: tlsParams, + token: token, + password: password) + } + + private func updateLastDiscoveredGateway(from gateways: [GatewayDiscoveryModel.DiscoveredGateway]) { + let defaults = UserDefaults.standard + let preferred = defaults.string(forKey: "gateway.preferredStableID")? + .trimmingCharacters(in: .whitespacesAndNewlines) ?? "" + let existingLast = defaults.string(forKey: "gateway.lastDiscoveredStableID")? + .trimmingCharacters(in: .whitespacesAndNewlines) ?? "" + + // Avoid overriding user intent (preferred/lastDiscovered are also set on manual Connect). + guard preferred.isEmpty, existingLast.isEmpty else { return } + guard let first = gateways.first else { return } + + defaults.set(first.stableID, forKey: "gateway.lastDiscoveredStableID") + GatewaySettingsStore.saveLastDiscoveredGatewayStableID(first.stableID) + } + + private func startAutoConnect( + url: URL, + gatewayStableID: String, + tls: GatewayTLSParams?, + token: String?, + password: String?) + { + guard let appModel else { return } + let connectOptions = self.makeConnectOptions() + + Task { [weak self] in + guard let self else { return } + await MainActor.run { + appModel.gatewayStatusText = "Connecting…" + } + appModel.connectToGateway( + url: url, + gatewayStableID: gatewayStableID, + tls: tls, + token: token, + password: password, + connectOptions: connectOptions) + } + } + + private func resolveDiscoveredTLSParams(gateway: GatewayDiscoveryModel.DiscoveredGateway) -> GatewayTLSParams? { + let stableID = gateway.stableID + let stored = GatewayTLSStore.loadFingerprint(stableID: stableID) + + if gateway.tlsEnabled || gateway.tlsFingerprintSha256 != nil || stored != nil { + return GatewayTLSParams( + required: true, + expectedFingerprint: gateway.tlsFingerprintSha256 ?? stored, + allowTOFU: stored == nil, + storeKey: stableID) + } + + return nil + } + + private func resolveManualTLSParams(stableID: String, tlsEnabled: Bool) -> GatewayTLSParams? { + let stored = GatewayTLSStore.loadFingerprint(stableID: stableID) + if tlsEnabled || stored != nil { + return GatewayTLSParams( + required: true, + expectedFingerprint: stored, + allowTOFU: stored == nil, + storeKey: stableID) + } + + return nil + } + + private func resolveGatewayHost(_ gateway: GatewayDiscoveryModel.DiscoveredGateway) -> String? { + if let lanHost = gateway.lanHost?.trimmingCharacters(in: .whitespacesAndNewlines), !lanHost.isEmpty { + return lanHost + } + if let tailnet = gateway.tailnetDns?.trimmingCharacters(in: .whitespacesAndNewlines), !tailnet.isEmpty { + return tailnet + } + return nil + } + + private func buildGatewayURL(host: String, port: Int, useTLS: Bool) -> URL? { + let scheme = useTLS ? "wss" : "ws" + var components = URLComponents() + components.scheme = scheme + components.host = host + components.port = port + return components.url + } + + private func manualStableID(host: String, port: Int) -> String { + "manual|\(host.lowercased())|\(port)" + } + + private func makeConnectOptions() -> GatewayConnectOptions { + let defaults = UserDefaults.standard + let displayName = self.resolvedDisplayName(defaults: defaults) + + return GatewayConnectOptions( + role: "node", + scopes: [], + caps: self.currentCaps(), + commands: self.currentCommands(), + permissions: [:], + clientId: "openclaw-ios", + clientMode: "node", + clientDisplayName: displayName) + } + + private func resolvedDisplayName(defaults: UserDefaults) -> String { + let key = "node.displayName" + let existing = defaults.string(forKey: key)?.trimmingCharacters(in: .whitespacesAndNewlines) ?? "" + if !existing.isEmpty, existing != "iOS Node" { return existing } + + let deviceName = UIDevice.current.name.trimmingCharacters(in: .whitespacesAndNewlines) + let candidate = deviceName.isEmpty ? "iOS Node" : deviceName + + if existing.isEmpty || existing == "iOS Node" { + defaults.set(candidate, forKey: key) + } + + return candidate + } + + private func currentCaps() -> [String] { + var caps = [OpenClawCapability.canvas.rawValue, OpenClawCapability.screen.rawValue] + + // Default-on: if the key doesn't exist yet, treat it as enabled. + let cameraEnabled = + UserDefaults.standard.object(forKey: "camera.enabled") == nil + ? true + : UserDefaults.standard.bool(forKey: "camera.enabled") + if cameraEnabled { caps.append(OpenClawCapability.camera.rawValue) } + + let voiceWakeEnabled = UserDefaults.standard.bool(forKey: VoiceWakePreferences.enabledKey) + if voiceWakeEnabled { caps.append(OpenClawCapability.voiceWake.rawValue) } + + let locationModeRaw = UserDefaults.standard.string(forKey: "location.enabledMode") ?? "off" + let locationMode = OpenClawLocationMode(rawValue: locationModeRaw) ?? .off + if locationMode != .off { caps.append(OpenClawCapability.location.rawValue) } + + return caps + } + + private func currentCommands() -> [String] { + var commands: [String] = [ + OpenClawCanvasCommand.present.rawValue, + OpenClawCanvasCommand.hide.rawValue, + OpenClawCanvasCommand.navigate.rawValue, + OpenClawCanvasCommand.evalJS.rawValue, + OpenClawCanvasCommand.snapshot.rawValue, + OpenClawCanvasA2UICommand.push.rawValue, + OpenClawCanvasA2UICommand.pushJSONL.rawValue, + OpenClawCanvasA2UICommand.reset.rawValue, + OpenClawScreenCommand.record.rawValue, + OpenClawSystemCommand.notify.rawValue, + OpenClawSystemCommand.which.rawValue, + OpenClawSystemCommand.run.rawValue, + OpenClawSystemCommand.execApprovalsGet.rawValue, + OpenClawSystemCommand.execApprovalsSet.rawValue, + ] + + let caps = Set(self.currentCaps()) + if caps.contains(OpenClawCapability.camera.rawValue) { + commands.append(OpenClawCameraCommand.list.rawValue) + commands.append(OpenClawCameraCommand.snap.rawValue) + commands.append(OpenClawCameraCommand.clip.rawValue) + } + if caps.contains(OpenClawCapability.location.rawValue) { + commands.append(OpenClawLocationCommand.get.rawValue) + } + + return commands + } + + private func platformString() -> String { + let v = ProcessInfo.processInfo.operatingSystemVersion + let name = switch UIDevice.current.userInterfaceIdiom { + case .pad: + "iPadOS" + case .phone: + "iOS" + default: + "iOS" + } + return "\(name) \(v.majorVersion).\(v.minorVersion).\(v.patchVersion)" + } + + private func deviceFamily() -> String { + switch UIDevice.current.userInterfaceIdiom { + case .pad: + "iPad" + case .phone: + "iPhone" + default: + "iOS" + } + } + + private func modelIdentifier() -> String { + var systemInfo = utsname() + uname(&systemInfo) + let machine = withUnsafeBytes(of: &systemInfo.machine) { ptr in + String(bytes: ptr.prefix { $0 != 0 }, encoding: .utf8) + } + let trimmed = machine?.trimmingCharacters(in: .whitespacesAndNewlines) ?? "" + return trimmed.isEmpty ? "unknown" : trimmed + } + + private func appVersion() -> String { + Bundle.main.infoDictionary?["CFBundleShortVersionString"] as? String ?? "dev" + } +} + +#if DEBUG +extension GatewayConnectionController { + func _test_resolvedDisplayName(defaults: UserDefaults) -> String { + self.resolvedDisplayName(defaults: defaults) + } + + func _test_currentCaps() -> [String] { + self.currentCaps() + } + + func _test_currentCommands() -> [String] { + self.currentCommands() + } + + func _test_platformString() -> String { + self.platformString() + } + + func _test_deviceFamily() -> String { + self.deviceFamily() + } + + func _test_modelIdentifier() -> String { + self.modelIdentifier() + } + + func _test_appVersion() -> String { + self.appVersion() + } + + func _test_setGateways(_ gateways: [GatewayDiscoveryModel.DiscoveredGateway]) { + self.gateways = gateways + } + + func _test_triggerAutoConnect() { + self.maybeAutoConnect() + } +} +#endif diff --git a/apps/ios/Sources/Gateway/GatewayDiscoveryDebugLogView.swift b/apps/ios/Sources/Gateway/GatewayDiscoveryDebugLogView.swift new file mode 100644 index 0000000000000000000000000000000000000000..8307225409ca7caefebf36d887816c89a96e8da1 --- /dev/null +++ b/apps/ios/Sources/Gateway/GatewayDiscoveryDebugLogView.swift @@ -0,0 +1,68 @@ +import SwiftUI +import UIKit + +struct GatewayDiscoveryDebugLogView: View { + @Environment(GatewayConnectionController.self) private var gatewayController + @AppStorage("gateway.discovery.debugLogs") private var debugLogsEnabled: Bool = false + + var body: some View { + List { + if !self.debugLogsEnabled { + Text("Enable “Discovery Debug Logs” to start collecting events.") + .foregroundStyle(.secondary) + } + + if self.gatewayController.discoveryDebugLog.isEmpty { + Text("No log entries yet.") + .foregroundStyle(.secondary) + } else { + ForEach(self.gatewayController.discoveryDebugLog) { entry in + VStack(alignment: .leading, spacing: 2) { + Text(Self.formatTime(entry.ts)) + .font(.caption) + .foregroundStyle(.secondary) + Text(entry.message) + .font(.callout) + .textSelection(.enabled) + } + .padding(.vertical, 4) + } + } + } + .navigationTitle("Discovery Logs") + .toolbar { + ToolbarItem(placement: .topBarTrailing) { + Button("Copy") { + UIPasteboard.general.string = self.formattedLog() + } + .disabled(self.gatewayController.discoveryDebugLog.isEmpty) + } + } + } + + private func formattedLog() -> String { + self.gatewayController.discoveryDebugLog + .map { "\(Self.formatISO($0.ts)) \($0.message)" } + .joined(separator: "\n") + } + + private static let timeFormatter: DateFormatter = { + let formatter = DateFormatter() + formatter.dateFormat = "HH:mm:ss" + return formatter + }() + + private static let isoFormatter: ISO8601DateFormatter = { + let formatter = ISO8601DateFormatter() + formatter.formatOptions = [.withInternetDateTime, .withFractionalSeconds] + return formatter + }() + + private static func formatTime(_ date: Date) -> String { + self.timeFormatter.string(from: date) + } + + private static func formatISO(_ date: Date) -> String { + self.isoFormatter.string(from: date) + } +} diff --git a/apps/ios/Sources/Gateway/GatewayDiscoveryModel.swift b/apps/ios/Sources/Gateway/GatewayDiscoveryModel.swift new file mode 100644 index 0000000000000000000000000000000000000000..223cfda5c908626fdf7cdc03f5f9101a6c01df8f --- /dev/null +++ b/apps/ios/Sources/Gateway/GatewayDiscoveryModel.swift @@ -0,0 +1,224 @@ +import OpenClawKit +import Foundation +import Network +import Observation + +@MainActor +@Observable +final class GatewayDiscoveryModel { + struct DebugLogEntry: Identifiable, Equatable { + var id = UUID() + var ts: Date + var message: String + } + + struct DiscoveredGateway: Identifiable, Equatable { + var id: String { self.stableID } + var name: String + var endpoint: NWEndpoint + var stableID: String + var debugID: String + var lanHost: String? + var tailnetDns: String? + var gatewayPort: Int? + var canvasPort: Int? + var tlsEnabled: Bool + var tlsFingerprintSha256: String? + var cliPath: String? + } + + var gateways: [DiscoveredGateway] = [] + var statusText: String = "Idle" + private(set) var debugLog: [DebugLogEntry] = [] + + private var browsers: [String: NWBrowser] = [:] + private var gatewaysByDomain: [String: [DiscoveredGateway]] = [:] + private var statesByDomain: [String: NWBrowser.State] = [:] + private var debugLoggingEnabled = false + private var lastStableIDs = Set() + + func setDebugLoggingEnabled(_ enabled: Bool) { + let wasEnabled = self.debugLoggingEnabled + self.debugLoggingEnabled = enabled + if !enabled { + self.debugLog = [] + } else if !wasEnabled { + self.appendDebugLog("debug logging enabled") + self.appendDebugLog("snapshot: status=\(self.statusText) gateways=\(self.gateways.count)") + } + } + + func start() { + if !self.browsers.isEmpty { return } + self.appendDebugLog("start()") + + for domain in OpenClawBonjour.gatewayServiceDomains { + let params = NWParameters.tcp + params.includePeerToPeer = true + let browser = NWBrowser( + for: .bonjour(type: OpenClawBonjour.gatewayServiceType, domain: domain), + using: params) + + browser.stateUpdateHandler = { [weak self] state in + Task { @MainActor in + guard let self else { return } + self.statesByDomain[domain] = state + self.updateStatusText() + self.appendDebugLog("state[\(domain)]: \(Self.prettyState(state))") + } + } + + browser.browseResultsChangedHandler = { [weak self] results, _ in + Task { @MainActor in + guard let self else { return } + self.gatewaysByDomain[domain] = results.compactMap { result -> DiscoveredGateway? in + switch result.endpoint { + case let .service(name, _, _, _): + let decodedName = BonjourEscapes.decode(name) + let txt = result.endpoint.txtRecord?.dictionary ?? [:] + let advertisedName = txt["displayName"] + let prettyAdvertised = advertisedName + .map(Self.prettifyInstanceName) + .flatMap { $0.isEmpty ? nil : $0 } + let prettyName = prettyAdvertised ?? Self.prettifyInstanceName(decodedName) + return DiscoveredGateway( + name: prettyName, + endpoint: result.endpoint, + stableID: GatewayEndpointID.stableID(result.endpoint), + debugID: GatewayEndpointID.prettyDescription(result.endpoint), + lanHost: Self.txtValue(txt, key: "lanHost"), + tailnetDns: Self.txtValue(txt, key: "tailnetDns"), + gatewayPort: Self.txtIntValue(txt, key: "gatewayPort"), + canvasPort: Self.txtIntValue(txt, key: "canvasPort"), + tlsEnabled: Self.txtBoolValue(txt, key: "gatewayTls"), + tlsFingerprintSha256: Self.txtValue(txt, key: "gatewayTlsSha256"), + cliPath: Self.txtValue(txt, key: "cliPath")) + default: + return nil + } + } + .sorted { $0.name.localizedCaseInsensitiveCompare($1.name) == .orderedAscending } + + self.recomputeGateways() + } + } + + self.browsers[domain] = browser + browser.start(queue: DispatchQueue(label: "bot.molt.ios.gateway-discovery.\(domain)")) + } + } + + func stop() { + self.appendDebugLog("stop()") + for browser in self.browsers.values { + browser.cancel() + } + self.browsers = [:] + self.gatewaysByDomain = [:] + self.statesByDomain = [:] + self.gateways = [] + self.statusText = "Stopped" + } + + private func recomputeGateways() { + let next = self.gatewaysByDomain.values + .flatMap(\.self) + .sorted { $0.name.localizedCaseInsensitiveCompare($1.name) == .orderedAscending } + + let nextIDs = Set(next.map(\.stableID)) + let added = nextIDs.subtracting(self.lastStableIDs) + let removed = self.lastStableIDs.subtracting(nextIDs) + if !added.isEmpty || !removed.isEmpty { + self.appendDebugLog("results: total=\(next.count) added=\(added.count) removed=\(removed.count)") + } + self.lastStableIDs = nextIDs + self.gateways = next + } + + private func updateStatusText() { + let states = Array(self.statesByDomain.values) + if states.isEmpty { + self.statusText = self.browsers.isEmpty ? "Idle" : "Setup" + return + } + + if let failed = states.first(where: { state in + if case .failed = state { return true } + return false + }) { + if case let .failed(err) = failed { + self.statusText = "Failed: \(err)" + return + } + } + + if let waiting = states.first(where: { state in + if case .waiting = state { return true } + return false + }) { + if case let .waiting(err) = waiting { + self.statusText = "Waiting: \(err)" + return + } + } + + if states.contains(where: { if case .ready = $0 { true } else { false } }) { + self.statusText = "Searching…" + return + } + + if states.contains(where: { if case .setup = $0 { true } else { false } }) { + self.statusText = "Setup" + return + } + + self.statusText = "Searching…" + } + + private static func prettyState(_ state: NWBrowser.State) -> String { + switch state { + case .setup: + "setup" + case .ready: + "ready" + case let .failed(err): + "failed (\(err))" + case .cancelled: + "cancelled" + case let .waiting(err): + "waiting (\(err))" + @unknown default: + "unknown" + } + } + + private func appendDebugLog(_ message: String) { + guard self.debugLoggingEnabled else { return } + self.debugLog.append(DebugLogEntry(ts: Date(), message: message)) + if self.debugLog.count > 200 { + self.debugLog.removeFirst(self.debugLog.count - 200) + } + } + + private static func prettifyInstanceName(_ decodedName: String) -> String { + let normalized = decodedName.split(whereSeparator: \.isWhitespace).joined(separator: " ") + let stripped = normalized.replacingOccurrences(of: " (OpenClaw)", with: "") + .replacingOccurrences(of: #"\s+\(\d+\)$"#, with: "", options: .regularExpression) + return stripped.trimmingCharacters(in: .whitespacesAndNewlines) + } + + private static func txtValue(_ dict: [String: String], key: String) -> String? { + let raw = dict[key]?.trimmingCharacters(in: .whitespacesAndNewlines) ?? "" + return raw.isEmpty ? nil : raw + } + + private static func txtIntValue(_ dict: [String: String], key: String) -> Int? { + guard let raw = self.txtValue(dict, key: key) else { return nil } + return Int(raw) + } + + private static func txtBoolValue(_ dict: [String: String], key: String) -> Bool { + guard let raw = self.txtValue(dict, key: key)?.lowercased() else { return false } + return raw == "1" || raw == "true" || raw == "yes" + } +} diff --git a/apps/ios/Sources/Gateway/GatewaySettingsStore.swift b/apps/ios/Sources/Gateway/GatewaySettingsStore.swift new file mode 100644 index 0000000000000000000000000000000000000000..4560dab788f0633658bb676924aeff2df6ce47fa --- /dev/null +++ b/apps/ios/Sources/Gateway/GatewaySettingsStore.swift @@ -0,0 +1,177 @@ +import Foundation + +enum GatewaySettingsStore { + private static let gatewayService = "ai.openclaw.gateway" + private static let nodeService = "ai.openclaw.node" + + private static let instanceIdDefaultsKey = "node.instanceId" + private static let preferredGatewayStableIDDefaultsKey = "gateway.preferredStableID" + private static let lastDiscoveredGatewayStableIDDefaultsKey = "gateway.lastDiscoveredStableID" + private static let manualEnabledDefaultsKey = "gateway.manual.enabled" + private static let manualHostDefaultsKey = "gateway.manual.host" + private static let manualPortDefaultsKey = "gateway.manual.port" + private static let manualTlsDefaultsKey = "gateway.manual.tls" + private static let discoveryDebugLogsDefaultsKey = "gateway.discovery.debugLogs" + + private static let instanceIdAccount = "instanceId" + private static let preferredGatewayStableIDAccount = "preferredStableID" + private static let lastDiscoveredGatewayStableIDAccount = "lastDiscoveredStableID" + + static func bootstrapPersistence() { + self.ensureStableInstanceID() + self.ensurePreferredGatewayStableID() + self.ensureLastDiscoveredGatewayStableID() + } + + static func loadStableInstanceID() -> String? { + if let value = KeychainStore.loadString(service: self.nodeService, account: self.instanceIdAccount)? + .trimmingCharacters(in: .whitespacesAndNewlines), + !value.isEmpty + { + return value + } + + return nil + } + + static func saveStableInstanceID(_ instanceId: String) { + _ = KeychainStore.saveString(instanceId, service: self.nodeService, account: self.instanceIdAccount) + } + + static func loadPreferredGatewayStableID() -> String? { + if let value = KeychainStore.loadString( + service: self.gatewayService, + account: self.preferredGatewayStableIDAccount + )?.trimmingCharacters(in: .whitespacesAndNewlines), + !value.isEmpty + { + return value + } + + return nil + } + + static func savePreferredGatewayStableID(_ stableID: String) { + _ = KeychainStore.saveString( + stableID, + service: self.gatewayService, + account: self.preferredGatewayStableIDAccount) + } + + static func loadLastDiscoveredGatewayStableID() -> String? { + if let value = KeychainStore.loadString( + service: self.gatewayService, + account: self.lastDiscoveredGatewayStableIDAccount + )?.trimmingCharacters(in: .whitespacesAndNewlines), + !value.isEmpty + { + return value + } + + return nil + } + + static func saveLastDiscoveredGatewayStableID(_ stableID: String) { + _ = KeychainStore.saveString( + stableID, + service: self.gatewayService, + account: self.lastDiscoveredGatewayStableIDAccount) + } + + static func loadGatewayToken(instanceId: String) -> String? { + let account = self.gatewayTokenAccount(instanceId: instanceId) + let token = KeychainStore.loadString(service: self.gatewayService, account: account)? + .trimmingCharacters(in: .whitespacesAndNewlines) + if token?.isEmpty == false { return token } + return nil + } + + static func saveGatewayToken(_ token: String, instanceId: String) { + _ = KeychainStore.saveString( + token, + service: self.gatewayService, + account: self.gatewayTokenAccount(instanceId: instanceId)) + } + + static func loadGatewayPassword(instanceId: String) -> String? { + KeychainStore.loadString( + service: self.gatewayService, + account: self.gatewayPasswordAccount(instanceId: instanceId))? + .trimmingCharacters(in: .whitespacesAndNewlines) + } + + static func saveGatewayPassword(_ password: String, instanceId: String) { + _ = KeychainStore.saveString( + password, + service: self.gatewayService, + account: self.gatewayPasswordAccount(instanceId: instanceId)) + } + + private static func gatewayTokenAccount(instanceId: String) -> String { + "gateway-token.\(instanceId)" + } + + private static func gatewayPasswordAccount(instanceId: String) -> String { + "gateway-password.\(instanceId)" + } + + private static func ensureStableInstanceID() { + let defaults = UserDefaults.standard + + if let existing = defaults.string(forKey: self.instanceIdDefaultsKey)? + .trimmingCharacters(in: .whitespacesAndNewlines), + !existing.isEmpty + { + if self.loadStableInstanceID() == nil { + self.saveStableInstanceID(existing) + } + return + } + + if let stored = self.loadStableInstanceID(), !stored.isEmpty { + defaults.set(stored, forKey: self.instanceIdDefaultsKey) + return + } + + let fresh = UUID().uuidString + self.saveStableInstanceID(fresh) + defaults.set(fresh, forKey: self.instanceIdDefaultsKey) + } + + private static func ensurePreferredGatewayStableID() { + let defaults = UserDefaults.standard + + if let existing = defaults.string(forKey: self.preferredGatewayStableIDDefaultsKey)? + .trimmingCharacters(in: .whitespacesAndNewlines), + !existing.isEmpty + { + if self.loadPreferredGatewayStableID() == nil { + self.savePreferredGatewayStableID(existing) + } + return + } + + if let stored = self.loadPreferredGatewayStableID(), !stored.isEmpty { + defaults.set(stored, forKey: self.preferredGatewayStableIDDefaultsKey) + } + } + + private static func ensureLastDiscoveredGatewayStableID() { + let defaults = UserDefaults.standard + + if let existing = defaults.string(forKey: self.lastDiscoveredGatewayStableIDDefaultsKey)? + .trimmingCharacters(in: .whitespacesAndNewlines), + !existing.isEmpty + { + if self.loadLastDiscoveredGatewayStableID() == nil { + self.saveLastDiscoveredGatewayStableID(existing) + } + return + } + + if let stored = self.loadLastDiscoveredGatewayStableID(), !stored.isEmpty { + defaults.set(stored, forKey: self.lastDiscoveredGatewayStableIDDefaultsKey) + } + } + +} diff --git a/apps/ios/Sources/Gateway/KeychainStore.swift b/apps/ios/Sources/Gateway/KeychainStore.swift new file mode 100644 index 0000000000000000000000000000000000000000..1377d8517ef64c4868398c1bb102067a10cbc2c5 --- /dev/null +++ b/apps/ios/Sources/Gateway/KeychainStore.swift @@ -0,0 +1,48 @@ +import Foundation +import Security + +enum KeychainStore { + static func loadString(service: String, account: String) -> String? { + let query: [String: Any] = [ + kSecClass as String: kSecClassGenericPassword, + kSecAttrService as String: service, + kSecAttrAccount as String: account, + kSecReturnData as String: true, + kSecMatchLimit as String: kSecMatchLimitOne, + ] + + var item: CFTypeRef? + let status = SecItemCopyMatching(query as CFDictionary, &item) + guard status == errSecSuccess, let data = item as? Data else { return nil } + return String(data: data, encoding: .utf8) + } + + static func saveString(_ value: String, service: String, account: String) -> Bool { + let data = Data(value.utf8) + let query: [String: Any] = [ + kSecClass as String: kSecClassGenericPassword, + kSecAttrService as String: service, + kSecAttrAccount as String: account, + ] + + let update: [String: Any] = [kSecValueData as String: data] + let status = SecItemUpdate(query as CFDictionary, update as CFDictionary) + if status == errSecSuccess { return true } + if status != errSecItemNotFound { return false } + + var insert = query + insert[kSecValueData as String] = data + insert[kSecAttrAccessible as String] = kSecAttrAccessibleAfterFirstUnlockThisDeviceOnly + return SecItemAdd(insert as CFDictionary, nil) == errSecSuccess + } + + static func delete(service: String, account: String) -> Bool { + let query: [String: Any] = [ + kSecClass as String: kSecClassGenericPassword, + kSecAttrService as String: service, + kSecAttrAccount as String: account, + ] + let status = SecItemDelete(query as CFDictionary) + return status == errSecSuccess || status == errSecItemNotFound + } +} diff --git a/apps/ios/Sources/Info.plist b/apps/ios/Sources/Info.plist new file mode 100644 index 0000000000000000000000000000000000000000..cf892f4ef92f7d8a00ec1a06d6ae5ee3cac9843e --- /dev/null +++ b/apps/ios/Sources/Info.plist @@ -0,0 +1,72 @@ + + + + + CFBundleDevelopmentRegion + $(DEVELOPMENT_LANGUAGE) + CFBundleDisplayName + OpenClaw + CFBundleExecutable + $(EXECUTABLE_NAME) + CFBundleIconName + AppIcon + CFBundleIdentifier + $(PRODUCT_BUNDLE_IDENTIFIER) + CFBundleInfoDictionaryVersion + 6.0 + CFBundleName + $(PRODUCT_NAME) + CFBundlePackageType + APPL + CFBundleShortVersionString + 2026.1.30 + CFBundleVersion + 20260129 + NSAppTransportSecurity + + NSAllowsArbitraryLoadsInWebContent + + + NSBonjourServices + + _openclaw-gw._tcp + + NSCameraUsageDescription + OpenClaw can capture photos or short video clips when requested via the gateway. + NSLocalNetworkUsageDescription + OpenClaw discovers and connects to your OpenClaw gateway on the local network. + NSLocationAlwaysAndWhenInUseUsageDescription + OpenClaw can share your location in the background when you enable Always. + NSLocationWhenInUseUsageDescription + OpenClaw uses your location when you allow location sharing. + NSMicrophoneUsageDescription + OpenClaw needs microphone access for voice wake. + NSSpeechRecognitionUsageDescription + OpenClaw uses on-device speech recognition for voice wake. + UIApplicationSceneManifest + + UIApplicationSupportsMultipleScenes + + + UIBackgroundModes + + audio + + UILaunchScreen + + UISupportedInterfaceOrientations + + UIInterfaceOrientationPortrait + UIInterfaceOrientationPortraitUpsideDown + UIInterfaceOrientationLandscapeLeft + UIInterfaceOrientationLandscapeRight + + UISupportedInterfaceOrientations~ipad + + UIInterfaceOrientationPortrait + UIInterfaceOrientationPortraitUpsideDown + UIInterfaceOrientationLandscapeLeft + UIInterfaceOrientationLandscapeRight + + + diff --git a/apps/ios/Sources/Location/LocationService.swift b/apps/ios/Sources/Location/LocationService.swift new file mode 100644 index 0000000000000000000000000000000000000000..99265d02e893fb714ffc1b0769ffde08c5f48074 --- /dev/null +++ b/apps/ios/Sources/Location/LocationService.swift @@ -0,0 +1,138 @@ +import OpenClawKit +import CoreLocation +import Foundation + +@MainActor +final class LocationService: NSObject, CLLocationManagerDelegate { + enum Error: Swift.Error { + case timeout + case unavailable + } + + private let manager = CLLocationManager() + private var authContinuation: CheckedContinuation? + private var locationContinuation: CheckedContinuation? + + override init() { + super.init() + self.manager.delegate = self + self.manager.desiredAccuracy = kCLLocationAccuracyBest + } + + func authorizationStatus() -> CLAuthorizationStatus { + self.manager.authorizationStatus + } + + func accuracyAuthorization() -> CLAccuracyAuthorization { + if #available(iOS 14.0, *) { + return self.manager.accuracyAuthorization + } + return .fullAccuracy + } + + func ensureAuthorization(mode: OpenClawLocationMode) async -> CLAuthorizationStatus { + guard CLLocationManager.locationServicesEnabled() else { return .denied } + + let status = self.manager.authorizationStatus + if status == .notDetermined { + self.manager.requestWhenInUseAuthorization() + let updated = await self.awaitAuthorizationChange() + if mode != .always { return updated } + } + + if mode == .always { + let current = self.manager.authorizationStatus + if current == .authorizedWhenInUse { + self.manager.requestAlwaysAuthorization() + return await self.awaitAuthorizationChange() + } + return current + } + + return self.manager.authorizationStatus + } + + func currentLocation( + params: OpenClawLocationGetParams, + desiredAccuracy: OpenClawLocationAccuracy, + maxAgeMs: Int?, + timeoutMs: Int?) async throws -> CLLocation + { + let now = Date() + if let maxAgeMs, + let cached = self.manager.location, + now.timeIntervalSince(cached.timestamp) * 1000 <= Double(maxAgeMs) + { + return cached + } + + self.manager.desiredAccuracy = Self.accuracyValue(desiredAccuracy) + let timeout = max(0, timeoutMs ?? 10000) + return try await self.withTimeout(timeoutMs: timeout) { + try await self.requestLocation() + } + } + + private func requestLocation() async throws -> CLLocation { + try await withCheckedThrowingContinuation { cont in + self.locationContinuation = cont + self.manager.requestLocation() + } + } + + private func awaitAuthorizationChange() async -> CLAuthorizationStatus { + await withCheckedContinuation { cont in + self.authContinuation = cont + } + } + + private func withTimeout( + timeoutMs: Int, + operation: @escaping @Sendable () async throws -> T) async throws -> T + { + try await AsyncTimeout.withTimeoutMs(timeoutMs: timeoutMs, onTimeout: { Error.timeout }, operation: operation) + } + + private static func accuracyValue(_ accuracy: OpenClawLocationAccuracy) -> CLLocationAccuracy { + switch accuracy { + case .coarse: + kCLLocationAccuracyKilometer + case .balanced: + kCLLocationAccuracyHundredMeters + case .precise: + kCLLocationAccuracyBest + } + } + + nonisolated func locationManagerDidChangeAuthorization(_ manager: CLLocationManager) { + let status = manager.authorizationStatus + Task { @MainActor in + if let cont = self.authContinuation { + self.authContinuation = nil + cont.resume(returning: status) + } + } + } + + nonisolated func locationManager(_ manager: CLLocationManager, didUpdateLocations locations: [CLLocation]) { + let locs = locations + Task { @MainActor in + guard let cont = self.locationContinuation else { return } + self.locationContinuation = nil + if let latest = locs.last { + cont.resume(returning: latest) + } else { + cont.resume(throwing: Error.unavailable) + } + } + } + + nonisolated func locationManager(_ manager: CLLocationManager, didFailWithError error: Swift.Error) { + let err = error + Task { @MainActor in + guard let cont = self.locationContinuation else { return } + self.locationContinuation = nil + cont.resume(throwing: err) + } + } +} diff --git a/apps/ios/Sources/Model/NodeAppModel.swift b/apps/ios/Sources/Model/NodeAppModel.swift new file mode 100644 index 0000000000000000000000000000000000000000..963318a8a2d2be967931c6eb5f221a7555c7bbe2 --- /dev/null +++ b/apps/ios/Sources/Model/NodeAppModel.swift @@ -0,0 +1,961 @@ +import OpenClawKit +import Network +import Observation +import SwiftUI +import UIKit + +@MainActor +@Observable +final class NodeAppModel { + enum CameraHUDKind { + case photo + case recording + case success + case error + } + + var isBackgrounded: Bool = false + let screen = ScreenController() + let camera = CameraController() + private let screenRecorder = ScreenRecordService() + var gatewayStatusText: String = "Offline" + var gatewayServerName: String? + var gatewayRemoteAddress: String? + var connectedGatewayID: String? + var seamColorHex: String? + var mainSessionKey: String = "main" + + private let gateway = GatewayNodeSession() + private var gatewayTask: Task? + private var voiceWakeSyncTask: Task? + @ObservationIgnored private var cameraHUDDismissTask: Task? + let voiceWake = VoiceWakeManager() + let talkMode = TalkModeManager() + private let locationService = LocationService() + private var lastAutoA2uiURL: String? + + private var gatewayConnected = false + var gatewaySession: GatewayNodeSession { self.gateway } + + var cameraHUDText: String? + var cameraHUDKind: CameraHUDKind? + var cameraFlashNonce: Int = 0 + var screenRecordActive: Bool = false + + init() { + self.voiceWake.configure { [weak self] cmd in + guard let self else { return } + let sessionKey = await MainActor.run { self.mainSessionKey } + do { + try await self.sendVoiceTranscript(text: cmd, sessionKey: sessionKey) + } catch { + // Best-effort only. + } + } + + let enabled = UserDefaults.standard.bool(forKey: "voiceWake.enabled") + self.voiceWake.setEnabled(enabled) + self.talkMode.attachGateway(self.gateway) + let talkEnabled = UserDefaults.standard.bool(forKey: "talk.enabled") + self.talkMode.setEnabled(talkEnabled) + + // Wire up deep links from canvas taps + self.screen.onDeepLink = { [weak self] url in + guard let self else { return } + Task { @MainActor in + await self.handleDeepLink(url: url) + } + } + + // Wire up A2UI action clicks (buttons, etc.) + self.screen.onA2UIAction = { [weak self] body in + guard let self else { return } + Task { @MainActor in + await self.handleCanvasA2UIAction(body: body) + } + } + } + + private func handleCanvasA2UIAction(body: [String: Any]) async { + let userActionAny = body["userAction"] ?? body + let userAction: [String: Any] = { + if let dict = userActionAny as? [String: Any] { return dict } + if let dict = userActionAny as? [AnyHashable: Any] { + return dict.reduce(into: [String: Any]()) { acc, pair in + guard let key = pair.key as? String else { return } + acc[key] = pair.value + } + } + return [:] + }() + guard !userAction.isEmpty else { return } + + guard let name = OpenClawCanvasA2UIAction.extractActionName(userAction) else { return } + let actionId: String = { + let id = (userAction["id"] as? String)?.trimmingCharacters(in: .whitespacesAndNewlines) ?? "" + return id.isEmpty ? UUID().uuidString : id + }() + + let surfaceId: String = { + let raw = (userAction["surfaceId"] as? String)?.trimmingCharacters(in: .whitespacesAndNewlines) ?? "" + return raw.isEmpty ? "main" : raw + }() + let sourceComponentId: String = { + let raw = (userAction[ + "sourceComponentId", + ] as? String)?.trimmingCharacters(in: .whitespacesAndNewlines) ?? "" + return raw.isEmpty ? "-" : raw + }() + + let host = UserDefaults.standard.string(forKey: "node.displayName") ?? UIDevice.current.name + let instanceId = (UserDefaults.standard.string(forKey: "node.instanceId") ?? "ios-node").lowercased() + let contextJSON = OpenClawCanvasA2UIAction.compactJSON(userAction["context"]) + let sessionKey = self.mainSessionKey + + let messageContext = OpenClawCanvasA2UIAction.AgentMessageContext( + actionName: name, + session: .init(key: sessionKey, surfaceId: surfaceId), + component: .init(id: sourceComponentId, host: host, instanceId: instanceId), + contextJSON: contextJSON) + let message = OpenClawCanvasA2UIAction.formatAgentMessage(messageContext) + + let ok: Bool + var errorText: String? + if await !self.isGatewayConnected() { + ok = false + errorText = "gateway not connected" + } else { + do { + try await self.sendAgentRequest(link: AgentDeepLink( + message: message, + sessionKey: sessionKey, + thinking: "low", + deliver: false, + to: nil, + channel: nil, + timeoutSeconds: nil, + key: actionId)) + ok = true + } catch { + ok = false + errorText = error.localizedDescription + } + } + + let js = OpenClawCanvasA2UIAction.jsDispatchA2UIActionStatus(actionId: actionId, ok: ok, error: errorText) + do { + _ = try await self.screen.eval(javaScript: js) + } catch { + // ignore + } + } + + private func resolveA2UIHostURL() async -> String? { + guard let raw = await self.gateway.currentCanvasHostUrl() else { return nil } + let trimmed = raw.trimmingCharacters(in: .whitespacesAndNewlines) + guard !trimmed.isEmpty, let base = URL(string: trimmed) else { return nil } + return base.appendingPathComponent("__openclaw__/a2ui/").absoluteString + "?platform=ios" + } + + private func showA2UIOnConnectIfNeeded() async { + guard let a2uiUrl = await self.resolveA2UIHostURL() else { return } + let current = self.screen.urlString.trimmingCharacters(in: .whitespacesAndNewlines) + if current.isEmpty || current == self.lastAutoA2uiURL { + self.screen.navigate(to: a2uiUrl) + self.lastAutoA2uiURL = a2uiUrl + } + } + + private func showLocalCanvasOnDisconnect() { + self.lastAutoA2uiURL = nil + self.screen.showDefaultCanvas() + } + + func setScenePhase(_ phase: ScenePhase) { + switch phase { + case .background: + self.isBackgrounded = true + case .active, .inactive: + self.isBackgrounded = false + @unknown default: + self.isBackgrounded = false + } + } + + func setVoiceWakeEnabled(_ enabled: Bool) { + self.voiceWake.setEnabled(enabled) + } + + func setTalkEnabled(_ enabled: Bool) { + self.talkMode.setEnabled(enabled) + } + + func requestLocationPermissions(mode: OpenClawLocationMode) async -> Bool { + guard mode != .off else { return true } + let status = await self.locationService.ensureAuthorization(mode: mode) + switch status { + case .authorizedAlways: + return true + case .authorizedWhenInUse: + return mode != .always + default: + return false + } + } + + func connectToGateway( + url: URL, + gatewayStableID: String, + tls: GatewayTLSParams?, + token: String?, + password: String?, + connectOptions: GatewayConnectOptions) + { + self.gatewayTask?.cancel() + self.gatewayServerName = nil + self.gatewayRemoteAddress = nil + let id = gatewayStableID.trimmingCharacters(in: .whitespacesAndNewlines) + self.connectedGatewayID = id.isEmpty ? url.absoluteString : id + self.gatewayConnected = false + self.voiceWakeSyncTask?.cancel() + self.voiceWakeSyncTask = nil + let sessionBox = tls.map { WebSocketSessionBox(session: GatewayTLSPinningSession(params: $0)) } + + self.gatewayTask = Task { + var attempt = 0 + while !Task.isCancelled { + await MainActor.run { + if attempt == 0 { + self.gatewayStatusText = "Connecting…" + } else { + self.gatewayStatusText = "Reconnecting…" + } + self.gatewayServerName = nil + self.gatewayRemoteAddress = nil + } + + do { + try await self.gateway.connect( + url: url, + token: token, + password: password, + connectOptions: connectOptions, + sessionBox: sessionBox, + onConnected: { [weak self] in + guard let self else { return } + await MainActor.run { + self.gatewayStatusText = "Connected" + self.gatewayServerName = url.host ?? "gateway" + self.gatewayConnected = true + } + if let addr = await self.gateway.currentRemoteAddress() { + await MainActor.run { + self.gatewayRemoteAddress = addr + } + } + await self.refreshBrandingFromGateway() + await self.startVoiceWakeSync() + await self.showA2UIOnConnectIfNeeded() + }, + onDisconnected: { [weak self] reason in + guard let self else { return } + await MainActor.run { + self.gatewayStatusText = "Disconnected" + self.gatewayRemoteAddress = nil + self.gatewayConnected = false + self.showLocalCanvasOnDisconnect() + self.gatewayStatusText = "Disconnected: \(reason)" + } + }, + onInvoke: { [weak self] req in + guard let self else { + return BridgeInvokeResponse( + id: req.id, + ok: false, + error: OpenClawNodeError( + code: .unavailable, + message: "UNAVAILABLE: node not ready")) + } + return await self.handleInvoke(req) + }) + + if Task.isCancelled { break } + attempt = 0 + try? await Task.sleep(nanoseconds: 1_000_000_000) + } catch { + if Task.isCancelled { break } + attempt += 1 + await MainActor.run { + self.gatewayStatusText = "Gateway error: \(error.localizedDescription)" + self.gatewayServerName = nil + self.gatewayRemoteAddress = nil + self.gatewayConnected = false + self.showLocalCanvasOnDisconnect() + } + let sleepSeconds = min(8.0, 0.5 * pow(1.7, Double(attempt))) + try? await Task.sleep(nanoseconds: UInt64(sleepSeconds * 1_000_000_000)) + } + } + + await MainActor.run { + self.gatewayStatusText = "Offline" + self.gatewayServerName = nil + self.gatewayRemoteAddress = nil + self.connectedGatewayID = nil + self.gatewayConnected = false + self.seamColorHex = nil + if !SessionKey.isCanonicalMainSessionKey(self.mainSessionKey) { + self.mainSessionKey = "main" + self.talkMode.updateMainSessionKey(self.mainSessionKey) + } + self.showLocalCanvasOnDisconnect() + } + } + } + + func disconnectGateway() { + self.gatewayTask?.cancel() + self.gatewayTask = nil + self.voiceWakeSyncTask?.cancel() + self.voiceWakeSyncTask = nil + Task { await self.gateway.disconnect() } + self.gatewayStatusText = "Offline" + self.gatewayServerName = nil + self.gatewayRemoteAddress = nil + self.connectedGatewayID = nil + self.gatewayConnected = false + self.seamColorHex = nil + if !SessionKey.isCanonicalMainSessionKey(self.mainSessionKey) { + self.mainSessionKey = "main" + self.talkMode.updateMainSessionKey(self.mainSessionKey) + } + self.showLocalCanvasOnDisconnect() + } + + private func applyMainSessionKey(_ key: String?) { + let trimmed = (key ?? "").trimmingCharacters(in: .whitespacesAndNewlines) + guard !trimmed.isEmpty else { return } + let current = self.mainSessionKey.trimmingCharacters(in: .whitespacesAndNewlines) + if SessionKey.isCanonicalMainSessionKey(current) { return } + if trimmed == current { return } + self.mainSessionKey = trimmed + self.talkMode.updateMainSessionKey(trimmed) + } + + var seamColor: Color { + Self.color(fromHex: self.seamColorHex) ?? Self.defaultSeamColor + } + + private static let defaultSeamColor = Color(red: 79 / 255.0, green: 122 / 255.0, blue: 154 / 255.0) + + private static func color(fromHex raw: String?) -> Color? { + let trimmed = (raw ?? "").trimmingCharacters(in: .whitespacesAndNewlines) + guard !trimmed.isEmpty else { return nil } + let hex = trimmed.hasPrefix("#") ? String(trimmed.dropFirst()) : trimmed + guard hex.count == 6, let value = Int(hex, radix: 16) else { return nil } + let r = Double((value >> 16) & 0xFF) / 255.0 + let g = Double((value >> 8) & 0xFF) / 255.0 + let b = Double(value & 0xFF) / 255.0 + return Color(red: r, green: g, blue: b) + } + + private func refreshBrandingFromGateway() async { + do { + let res = try await self.gateway.request(method: "config.get", paramsJSON: "{}", timeoutSeconds: 8) + guard let json = try JSONSerialization.jsonObject(with: res) as? [String: Any] else { return } + guard let config = json["config"] as? [String: Any] else { return } + let ui = config["ui"] as? [String: Any] + let raw = (ui?["seamColor"] as? String)?.trimmingCharacters(in: .whitespacesAndNewlines) ?? "" + let session = config["session"] as? [String: Any] + let mainKey = SessionKey.normalizeMainKey(session?["mainKey"] as? String) + await MainActor.run { + self.seamColorHex = raw.isEmpty ? nil : raw + if !SessionKey.isCanonicalMainSessionKey(self.mainSessionKey) { + self.mainSessionKey = mainKey + self.talkMode.updateMainSessionKey(mainKey) + } + } + } catch { + // ignore + } + } + + func setGlobalWakeWords(_ words: [String]) async { + let sanitized = VoiceWakePreferences.sanitizeTriggerWords(words) + + struct Payload: Codable { + var triggers: [String] + } + let payload = Payload(triggers: sanitized) + guard let data = try? JSONEncoder().encode(payload), + let json = String(data: data, encoding: .utf8) + else { return } + + do { + _ = try await self.gateway.request(method: "voicewake.set", paramsJSON: json, timeoutSeconds: 12) + } catch { + // Best-effort only. + } + } + + private func startVoiceWakeSync() async { + self.voiceWakeSyncTask?.cancel() + self.voiceWakeSyncTask = Task { [weak self] in + guard let self else { return } + + await self.refreshWakeWordsFromGateway() + + let stream = await self.gateway.subscribeServerEvents(bufferingNewest: 200) + for await evt in stream { + if Task.isCancelled { return } + guard evt.event == "voicewake.changed" else { continue } + guard let payload = evt.payload else { continue } + struct Payload: Decodable { var triggers: [String] } + guard let decoded = try? GatewayPayloadDecoding.decode(payload, as: Payload.self) else { continue } + let triggers = VoiceWakePreferences.sanitizeTriggerWords(decoded.triggers) + VoiceWakePreferences.saveTriggerWords(triggers) + } + } + } + + private func refreshWakeWordsFromGateway() async { + do { + let data = try await self.gateway.request(method: "voicewake.get", paramsJSON: "{}", timeoutSeconds: 8) + guard let triggers = VoiceWakePreferences.decodeGatewayTriggers(from: data) else { return } + VoiceWakePreferences.saveTriggerWords(triggers) + } catch { + // Best-effort only. + } + } + + func sendVoiceTranscript(text: String, sessionKey: String?) async throws { + if await !self.isGatewayConnected() { + throw NSError(domain: "Gateway", code: 10, userInfo: [ + NSLocalizedDescriptionKey: "Gateway not connected", + ]) + } + struct Payload: Codable { + var text: String + var sessionKey: String? + } + let payload = Payload(text: text, sessionKey: sessionKey) + let data = try JSONEncoder().encode(payload) + guard let json = String(bytes: data, encoding: .utf8) else { + throw NSError(domain: "NodeAppModel", code: 1, userInfo: [ + NSLocalizedDescriptionKey: "Failed to encode voice transcript payload as UTF-8", + ]) + } + await self.gateway.sendEvent(event: "voice.transcript", payloadJSON: json) + } + + func handleDeepLink(url: URL) async { + guard let route = DeepLinkParser.parse(url) else { return } + + switch route { + case let .agent(link): + await self.handleAgentDeepLink(link, originalURL: url) + } + } + + private func handleAgentDeepLink(_ link: AgentDeepLink, originalURL: URL) async { + let message = link.message.trimmingCharacters(in: .whitespacesAndNewlines) + guard !message.isEmpty else { return } + + if message.count > 20000 { + self.screen.errorText = "Deep link too large (message exceeds 20,000 characters)." + return + } + + guard await self.isGatewayConnected() else { + self.screen.errorText = "Gateway not connected (cannot forward deep link)." + return + } + + do { + try await self.sendAgentRequest(link: link) + self.screen.errorText = nil + } catch { + self.screen.errorText = "Agent request failed: \(error.localizedDescription)" + } + } + + private func sendAgentRequest(link: AgentDeepLink) async throws { + if link.message.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty { + throw NSError(domain: "DeepLink", code: 1, userInfo: [ + NSLocalizedDescriptionKey: "invalid agent message", + ]) + } + + // iOS gateway forwards to the gateway; no local auth prompts here. + // (Key-based unattended auth is handled on macOS for openclaw:// links.) + let data = try JSONEncoder().encode(link) + guard let json = String(bytes: data, encoding: .utf8) else { + throw NSError(domain: "NodeAppModel", code: 2, userInfo: [ + NSLocalizedDescriptionKey: "Failed to encode agent request payload as UTF-8", + ]) + } + await self.gateway.sendEvent(event: "agent.request", payloadJSON: json) + } + + private func isGatewayConnected() async -> Bool { + self.gatewayConnected + } + + private func handleInvoke(_ req: BridgeInvokeRequest) async -> BridgeInvokeResponse { + let command = req.command + + if self.isBackgrounded, self.isBackgroundRestricted(command) { + return BridgeInvokeResponse( + id: req.id, + ok: false, + error: OpenClawNodeError( + code: .backgroundUnavailable, + message: "NODE_BACKGROUND_UNAVAILABLE: canvas/camera/screen commands require foreground")) + } + + if command.hasPrefix("camera."), !self.isCameraEnabled() { + return BridgeInvokeResponse( + id: req.id, + ok: false, + error: OpenClawNodeError( + code: .unavailable, + message: "CAMERA_DISABLED: enable Camera in iOS Settings → Camera → Allow Camera")) + } + + do { + switch command { + case OpenClawLocationCommand.get.rawValue: + return try await self.handleLocationInvoke(req) + case OpenClawCanvasCommand.present.rawValue, + OpenClawCanvasCommand.hide.rawValue, + OpenClawCanvasCommand.navigate.rawValue, + OpenClawCanvasCommand.evalJS.rawValue, + OpenClawCanvasCommand.snapshot.rawValue: + return try await self.handleCanvasInvoke(req) + case OpenClawCanvasA2UICommand.reset.rawValue, + OpenClawCanvasA2UICommand.push.rawValue, + OpenClawCanvasA2UICommand.pushJSONL.rawValue: + return try await self.handleCanvasA2UIInvoke(req) + case OpenClawCameraCommand.list.rawValue, + OpenClawCameraCommand.snap.rawValue, + OpenClawCameraCommand.clip.rawValue: + return try await self.handleCameraInvoke(req) + case OpenClawScreenCommand.record.rawValue: + return try await self.handleScreenRecordInvoke(req) + default: + return BridgeInvokeResponse( + id: req.id, + ok: false, + error: OpenClawNodeError(code: .invalidRequest, message: "INVALID_REQUEST: unknown command")) + } + } catch { + if command.hasPrefix("camera.") { + let text = (error as? LocalizedError)?.errorDescription ?? error.localizedDescription + self.showCameraHUD(text: text, kind: .error, autoHideSeconds: 2.2) + } + return BridgeInvokeResponse( + id: req.id, + ok: false, + error: OpenClawNodeError(code: .unavailable, message: error.localizedDescription)) + } + } + + private func isBackgroundRestricted(_ command: String) -> Bool { + command.hasPrefix("canvas.") || command.hasPrefix("camera.") || command.hasPrefix("screen.") + } + + private func handleLocationInvoke(_ req: BridgeInvokeRequest) async throws -> BridgeInvokeResponse { + let mode = self.locationMode() + guard mode != .off else { + return BridgeInvokeResponse( + id: req.id, + ok: false, + error: OpenClawNodeError( + code: .unavailable, + message: "LOCATION_DISABLED: enable Location in Settings")) + } + if self.isBackgrounded, mode != .always { + return BridgeInvokeResponse( + id: req.id, + ok: false, + error: OpenClawNodeError( + code: .backgroundUnavailable, + message: "LOCATION_BACKGROUND_UNAVAILABLE: background location requires Always")) + } + let params = (try? Self.decodeParams(OpenClawLocationGetParams.self, from: req.paramsJSON)) ?? + OpenClawLocationGetParams() + let desired = params.desiredAccuracy ?? + (self.isLocationPreciseEnabled() ? .precise : .balanced) + let status = self.locationService.authorizationStatus() + if status != .authorizedAlways, status != .authorizedWhenInUse { + return BridgeInvokeResponse( + id: req.id, + ok: false, + error: OpenClawNodeError( + code: .unavailable, + message: "LOCATION_PERMISSION_REQUIRED: grant Location permission")) + } + if self.isBackgrounded, status != .authorizedAlways { + return BridgeInvokeResponse( + id: req.id, + ok: false, + error: OpenClawNodeError( + code: .unavailable, + message: "LOCATION_PERMISSION_REQUIRED: enable Always for background access")) + } + let location = try await self.locationService.currentLocation( + params: params, + desiredAccuracy: desired, + maxAgeMs: params.maxAgeMs, + timeoutMs: params.timeoutMs) + let isPrecise = self.locationService.accuracyAuthorization() == .fullAccuracy + let payload = OpenClawLocationPayload( + lat: location.coordinate.latitude, + lon: location.coordinate.longitude, + accuracyMeters: location.horizontalAccuracy, + altitudeMeters: location.verticalAccuracy >= 0 ? location.altitude : nil, + speedMps: location.speed >= 0 ? location.speed : nil, + headingDeg: location.course >= 0 ? location.course : nil, + timestamp: ISO8601DateFormatter().string(from: location.timestamp), + isPrecise: isPrecise, + source: nil) + let json = try Self.encodePayload(payload) + return BridgeInvokeResponse(id: req.id, ok: true, payloadJSON: json) + } + + private func handleCanvasInvoke(_ req: BridgeInvokeRequest) async throws -> BridgeInvokeResponse { + switch req.command { + case OpenClawCanvasCommand.present.rawValue: + let params = (try? Self.decodeParams(OpenClawCanvasPresentParams.self, from: req.paramsJSON)) ?? + OpenClawCanvasPresentParams() + let url = params.url?.trimmingCharacters(in: .whitespacesAndNewlines) ?? "" + if url.isEmpty { + self.screen.showDefaultCanvas() + } else { + self.screen.navigate(to: url) + } + return BridgeInvokeResponse(id: req.id, ok: true) + case OpenClawCanvasCommand.hide.rawValue: + return BridgeInvokeResponse(id: req.id, ok: true) + case OpenClawCanvasCommand.navigate.rawValue: + let params = try Self.decodeParams(OpenClawCanvasNavigateParams.self, from: req.paramsJSON) + self.screen.navigate(to: params.url) + return BridgeInvokeResponse(id: req.id, ok: true) + case OpenClawCanvasCommand.evalJS.rawValue: + let params = try Self.decodeParams(OpenClawCanvasEvalParams.self, from: req.paramsJSON) + let result = try await self.screen.eval(javaScript: params.javaScript) + let payload = try Self.encodePayload(["result": result]) + return BridgeInvokeResponse(id: req.id, ok: true, payloadJSON: payload) + case OpenClawCanvasCommand.snapshot.rawValue: + let params = try? Self.decodeParams(OpenClawCanvasSnapshotParams.self, from: req.paramsJSON) + let format = params?.format ?? .jpeg + let maxWidth: CGFloat? = { + if let raw = params?.maxWidth, raw > 0 { return CGFloat(raw) } + // Keep default snapshots comfortably below the gateway client's maxPayload. + // For full-res, clients should explicitly request a larger maxWidth. + return switch format { + case .png: 900 + case .jpeg: 1600 + } + }() + let base64 = try await self.screen.snapshotBase64( + maxWidth: maxWidth, + format: format, + quality: params?.quality) + let payload = try Self.encodePayload([ + "format": format == .jpeg ? "jpeg" : "png", + "base64": base64, + ]) + return BridgeInvokeResponse(id: req.id, ok: true, payloadJSON: payload) + default: + return BridgeInvokeResponse( + id: req.id, + ok: false, + error: OpenClawNodeError(code: .invalidRequest, message: "INVALID_REQUEST: unknown command")) + } + } + + private func handleCanvasA2UIInvoke(_ req: BridgeInvokeRequest) async throws -> BridgeInvokeResponse { + let command = req.command + switch command { + case OpenClawCanvasA2UICommand.reset.rawValue: + guard let a2uiUrl = await self.resolveA2UIHostURL() else { + return BridgeInvokeResponse( + id: req.id, + ok: false, + error: OpenClawNodeError( + code: .unavailable, + message: "A2UI_HOST_NOT_CONFIGURED: gateway did not advertise canvas host")) + } + self.screen.navigate(to: a2uiUrl) + if await !self.screen.waitForA2UIReady(timeoutMs: 5000) { + return BridgeInvokeResponse( + id: req.id, + ok: false, + error: OpenClawNodeError( + code: .unavailable, + message: "A2UI_HOST_UNAVAILABLE: A2UI host not reachable")) + } + + let json = try await self.screen.eval(javaScript: """ + (() => { + const host = globalThis.openclawA2UI; + if (!host) return JSON.stringify({ ok: false, error: "missing openclawA2UI" }); + return JSON.stringify(host.reset()); + })() + """) + return BridgeInvokeResponse(id: req.id, ok: true, payloadJSON: json) + case OpenClawCanvasA2UICommand.push.rawValue, OpenClawCanvasA2UICommand.pushJSONL.rawValue: + let messages: [AnyCodable] + if command == OpenClawCanvasA2UICommand.pushJSONL.rawValue { + let params = try Self.decodeParams(OpenClawCanvasA2UIPushJSONLParams.self, from: req.paramsJSON) + messages = try OpenClawCanvasA2UIJSONL.decodeMessagesFromJSONL(params.jsonl) + } else { + do { + let params = try Self.decodeParams(OpenClawCanvasA2UIPushParams.self, from: req.paramsJSON) + messages = params.messages + } catch { + // Be forgiving: some clients still send JSONL payloads to `canvas.a2ui.push`. + let params = try Self.decodeParams(OpenClawCanvasA2UIPushJSONLParams.self, from: req.paramsJSON) + messages = try OpenClawCanvasA2UIJSONL.decodeMessagesFromJSONL(params.jsonl) + } + } + + guard let a2uiUrl = await self.resolveA2UIHostURL() else { + return BridgeInvokeResponse( + id: req.id, + ok: false, + error: OpenClawNodeError( + code: .unavailable, + message: "A2UI_HOST_NOT_CONFIGURED: gateway did not advertise canvas host")) + } + self.screen.navigate(to: a2uiUrl) + if await !self.screen.waitForA2UIReady(timeoutMs: 5000) { + return BridgeInvokeResponse( + id: req.id, + ok: false, + error: OpenClawNodeError( + code: .unavailable, + message: "A2UI_HOST_UNAVAILABLE: A2UI host not reachable")) + } + + let messagesJSON = try OpenClawCanvasA2UIJSONL.encodeMessagesJSONArray(messages) + let js = """ + (() => { + try { + const host = globalThis.openclawA2UI; + if (!host) return JSON.stringify({ ok: false, error: "missing openclawA2UI" }); + const messages = \(messagesJSON); + return JSON.stringify(host.applyMessages(messages)); + } catch (e) { + return JSON.stringify({ ok: false, error: String(e?.message ?? e) }); + } + })() + """ + let resultJSON = try await self.screen.eval(javaScript: js) + return BridgeInvokeResponse(id: req.id, ok: true, payloadJSON: resultJSON) + default: + return BridgeInvokeResponse( + id: req.id, + ok: false, + error: OpenClawNodeError(code: .invalidRequest, message: "INVALID_REQUEST: unknown command")) + } + } + + private func handleCameraInvoke(_ req: BridgeInvokeRequest) async throws -> BridgeInvokeResponse { + switch req.command { + case OpenClawCameraCommand.list.rawValue: + let devices = await self.camera.listDevices() + struct Payload: Codable { + var devices: [CameraController.CameraDeviceInfo] + } + let payload = try Self.encodePayload(Payload(devices: devices)) + return BridgeInvokeResponse(id: req.id, ok: true, payloadJSON: payload) + case OpenClawCameraCommand.snap.rawValue: + self.showCameraHUD(text: "Taking photo…", kind: .photo) + self.triggerCameraFlash() + let params = (try? Self.decodeParams(OpenClawCameraSnapParams.self, from: req.paramsJSON)) ?? + OpenClawCameraSnapParams() + let res = try await self.camera.snap(params: params) + + struct Payload: Codable { + var format: String + var base64: String + var width: Int + var height: Int + } + let payload = try Self.encodePayload(Payload( + format: res.format, + base64: res.base64, + width: res.width, + height: res.height)) + self.showCameraHUD(text: "Photo captured", kind: .success, autoHideSeconds: 1.6) + return BridgeInvokeResponse(id: req.id, ok: true, payloadJSON: payload) + case OpenClawCameraCommand.clip.rawValue: + let params = (try? Self.decodeParams(OpenClawCameraClipParams.self, from: req.paramsJSON)) ?? + OpenClawCameraClipParams() + + let suspended = (params.includeAudio ?? true) ? self.voiceWake.suspendForExternalAudioCapture() : false + defer { self.voiceWake.resumeAfterExternalAudioCapture(wasSuspended: suspended) } + + self.showCameraHUD(text: "Recording…", kind: .recording) + let res = try await self.camera.clip(params: params) + + struct Payload: Codable { + var format: String + var base64: String + var durationMs: Int + var hasAudio: Bool + } + let payload = try Self.encodePayload(Payload( + format: res.format, + base64: res.base64, + durationMs: res.durationMs, + hasAudio: res.hasAudio)) + self.showCameraHUD(text: "Clip captured", kind: .success, autoHideSeconds: 1.8) + return BridgeInvokeResponse(id: req.id, ok: true, payloadJSON: payload) + default: + return BridgeInvokeResponse( + id: req.id, + ok: false, + error: OpenClawNodeError(code: .invalidRequest, message: "INVALID_REQUEST: unknown command")) + } + } + + private func handleScreenRecordInvoke(_ req: BridgeInvokeRequest) async throws -> BridgeInvokeResponse { + let params = (try? Self.decodeParams(OpenClawScreenRecordParams.self, from: req.paramsJSON)) ?? + OpenClawScreenRecordParams() + if let format = params.format, format.lowercased() != "mp4" { + throw NSError(domain: "Screen", code: 30, userInfo: [ + NSLocalizedDescriptionKey: "INVALID_REQUEST: screen format must be mp4", + ]) + } + // Status pill mirrors screen recording state so it stays visible without overlay stacking. + self.screenRecordActive = true + defer { self.screenRecordActive = false } + let path = try await self.screenRecorder.record( + screenIndex: params.screenIndex, + durationMs: params.durationMs, + fps: params.fps, + includeAudio: params.includeAudio, + outPath: nil) + defer { try? FileManager().removeItem(atPath: path) } + let data = try Data(contentsOf: URL(fileURLWithPath: path)) + struct Payload: Codable { + var format: String + var base64: String + var durationMs: Int? + var fps: Double? + var screenIndex: Int? + var hasAudio: Bool + } + let payload = try Self.encodePayload(Payload( + format: "mp4", + base64: data.base64EncodedString(), + durationMs: params.durationMs, + fps: params.fps, + screenIndex: params.screenIndex, + hasAudio: params.includeAudio ?? true)) + return BridgeInvokeResponse(id: req.id, ok: true, payloadJSON: payload) + } + +} + +private extension NodeAppModel { + func locationMode() -> OpenClawLocationMode { + let raw = UserDefaults.standard.string(forKey: "location.enabledMode") ?? "off" + return OpenClawLocationMode(rawValue: raw) ?? .off + } + + func isLocationPreciseEnabled() -> Bool { + if UserDefaults.standard.object(forKey: "location.preciseEnabled") == nil { return true } + return UserDefaults.standard.bool(forKey: "location.preciseEnabled") + } + + static func decodeParams(_ type: T.Type, from json: String?) throws -> T { + guard let json, let data = json.data(using: .utf8) else { + throw NSError(domain: "Gateway", code: 20, userInfo: [ + NSLocalizedDescriptionKey: "INVALID_REQUEST: paramsJSON required", + ]) + } + return try JSONDecoder().decode(type, from: data) + } + + static func encodePayload(_ obj: some Encodable) throws -> String { + let data = try JSONEncoder().encode(obj) + guard let json = String(bytes: data, encoding: .utf8) else { + throw NSError(domain: "NodeAppModel", code: 21, userInfo: [ + NSLocalizedDescriptionKey: "Failed to encode payload as UTF-8", + ]) + } + return json + } + + func isCameraEnabled() -> Bool { + // Default-on: if the key doesn't exist yet, treat it as enabled. + if UserDefaults.standard.object(forKey: "camera.enabled") == nil { return true } + return UserDefaults.standard.bool(forKey: "camera.enabled") + } + + func triggerCameraFlash() { + self.cameraFlashNonce &+= 1 + } + + func showCameraHUD(text: String, kind: CameraHUDKind, autoHideSeconds: Double? = nil) { + self.cameraHUDDismissTask?.cancel() + + withAnimation(.spring(response: 0.25, dampingFraction: 0.85)) { + self.cameraHUDText = text + self.cameraHUDKind = kind + } + + guard let autoHideSeconds else { return } + self.cameraHUDDismissTask = Task { @MainActor in + try? await Task.sleep(nanoseconds: UInt64(autoHideSeconds * 1_000_000_000)) + withAnimation(.easeOut(duration: 0.25)) { + self.cameraHUDText = nil + self.cameraHUDKind = nil + } + } + } +} + +#if DEBUG +extension NodeAppModel { + func _test_handleInvoke(_ req: BridgeInvokeRequest) async -> BridgeInvokeResponse { + await self.handleInvoke(req) + } + + static func _test_decodeParams(_ type: T.Type, from json: String?) throws -> T { + try self.decodeParams(type, from: json) + } + + static func _test_encodePayload(_ obj: some Encodable) throws -> String { + try self.encodePayload(obj) + } + + func _test_isCameraEnabled() -> Bool { + self.isCameraEnabled() + } + + func _test_triggerCameraFlash() { + self.triggerCameraFlash() + } + + func _test_showCameraHUD(text: String, kind: CameraHUDKind, autoHideSeconds: Double? = nil) { + self.showCameraHUD(text: text, kind: kind, autoHideSeconds: autoHideSeconds) + } + + func _test_handleCanvasA2UIAction(body: [String: Any]) async { + await self.handleCanvasA2UIAction(body: body) + } + + func _test_resolveA2UIHostURL() async -> String? { + await self.resolveA2UIHostURL() + } + + func _test_showLocalCanvasOnDisconnect() { + self.showLocalCanvasOnDisconnect() + } +} +#endif diff --git a/apps/ios/Sources/OpenClawApp.swift b/apps/ios/Sources/OpenClawApp.swift new file mode 100644 index 0000000000000000000000000000000000000000..8ad23ae20a10a1066d77840fc0a69683b6aa1da2 --- /dev/null +++ b/apps/ios/Sources/OpenClawApp.swift @@ -0,0 +1,31 @@ +import SwiftUI + +@main +struct OpenClawApp: App { + @State private var appModel: NodeAppModel + @State private var gatewayController: GatewayConnectionController + @Environment(\.scenePhase) private var scenePhase + + init() { + GatewaySettingsStore.bootstrapPersistence() + let appModel = NodeAppModel() + _appModel = State(initialValue: appModel) + _gatewayController = State(initialValue: GatewayConnectionController(appModel: appModel)) + } + + var body: some Scene { + WindowGroup { + RootCanvas() + .environment(self.appModel) + .environment(self.appModel.voiceWake) + .environment(self.gatewayController) + .onOpenURL { url in + Task { await self.appModel.handleDeepLink(url: url) } + } + .onChange(of: self.scenePhase) { _, newValue in + self.appModel.setScenePhase(newValue) + self.gatewayController.setScenePhase(newValue) + } + } + } +} diff --git a/apps/ios/Sources/RootCanvas.swift b/apps/ios/Sources/RootCanvas.swift new file mode 100644 index 0000000000000000000000000000000000000000..93cb816273c2c5f055074e2babd555645507a005 --- /dev/null +++ b/apps/ios/Sources/RootCanvas.swift @@ -0,0 +1,341 @@ +import SwiftUI +import UIKit + +struct RootCanvas: View { + @Environment(NodeAppModel.self) private var appModel + @Environment(VoiceWakeManager.self) private var voiceWake + @Environment(\.colorScheme) private var systemColorScheme + @Environment(\.scenePhase) private var scenePhase + @AppStorage(VoiceWakePreferences.enabledKey) private var voiceWakeEnabled: Bool = false + @AppStorage("screen.preventSleep") private var preventSleep: Bool = true + @AppStorage("canvas.debugStatusEnabled") private var canvasDebugStatusEnabled: Bool = false + @State private var presentedSheet: PresentedSheet? + @State private var voiceWakeToastText: String? + @State private var toastDismissTask: Task? + + private enum PresentedSheet: Identifiable { + case settings + case chat + + var id: Int { + switch self { + case .settings: 0 + case .chat: 1 + } + } + } + + var body: some View { + ZStack { + CanvasContent( + systemColorScheme: self.systemColorScheme, + gatewayStatus: self.gatewayStatus, + voiceWakeEnabled: self.voiceWakeEnabled, + voiceWakeToastText: self.voiceWakeToastText, + cameraHUDText: self.appModel.cameraHUDText, + cameraHUDKind: self.appModel.cameraHUDKind, + openChat: { + self.presentedSheet = .chat + }, + openSettings: { + self.presentedSheet = .settings + }) + .preferredColorScheme(.dark) + + if self.appModel.cameraFlashNonce != 0 { + CameraFlashOverlay(nonce: self.appModel.cameraFlashNonce) + } + } + .sheet(item: self.$presentedSheet) { sheet in + switch sheet { + case .settings: + SettingsTab() + case .chat: + ChatSheet( + gateway: self.appModel.gatewaySession, + sessionKey: self.appModel.mainSessionKey, + userAccent: self.appModel.seamColor) + } + } + .onAppear { self.updateIdleTimer() } + .onChange(of: self.preventSleep) { _, _ in self.updateIdleTimer() } + .onChange(of: self.scenePhase) { _, _ in self.updateIdleTimer() } + .onAppear { self.updateCanvasDebugStatus() } + .onChange(of: self.canvasDebugStatusEnabled) { _, _ in self.updateCanvasDebugStatus() } + .onChange(of: self.appModel.gatewayStatusText) { _, _ in self.updateCanvasDebugStatus() } + .onChange(of: self.appModel.gatewayServerName) { _, _ in self.updateCanvasDebugStatus() } + .onChange(of: self.appModel.gatewayRemoteAddress) { _, _ in self.updateCanvasDebugStatus() } + .onChange(of: self.voiceWake.lastTriggeredCommand) { _, newValue in + guard let newValue else { return } + let trimmed = newValue.trimmingCharacters(in: .whitespacesAndNewlines) + guard !trimmed.isEmpty else { return } + + self.toastDismissTask?.cancel() + withAnimation(.spring(response: 0.25, dampingFraction: 0.85)) { + self.voiceWakeToastText = trimmed + } + + self.toastDismissTask = Task { + try? await Task.sleep(nanoseconds: 2_300_000_000) + await MainActor.run { + withAnimation(.easeOut(duration: 0.25)) { + self.voiceWakeToastText = nil + } + } + } + } + .onDisappear { + UIApplication.shared.isIdleTimerDisabled = false + self.toastDismissTask?.cancel() + self.toastDismissTask = nil + } + } + + private var gatewayStatus: StatusPill.GatewayState { + if self.appModel.gatewayServerName != nil { return .connected } + + let text = self.appModel.gatewayStatusText.trimmingCharacters(in: .whitespacesAndNewlines) + if text.localizedCaseInsensitiveContains("connecting") || + text.localizedCaseInsensitiveContains("reconnecting") + { + return .connecting + } + + if text.localizedCaseInsensitiveContains("error") { + return .error + } + + return .disconnected + } + + private func updateIdleTimer() { + UIApplication.shared.isIdleTimerDisabled = (self.scenePhase == .active && self.preventSleep) + } + + private func updateCanvasDebugStatus() { + self.appModel.screen.setDebugStatusEnabled(self.canvasDebugStatusEnabled) + guard self.canvasDebugStatusEnabled else { return } + let title = self.appModel.gatewayStatusText.trimmingCharacters(in: .whitespacesAndNewlines) + let subtitle = self.appModel.gatewayServerName ?? self.appModel.gatewayRemoteAddress + self.appModel.screen.updateDebugStatus(title: title, subtitle: subtitle) + } +} + +private struct CanvasContent: View { + @Environment(NodeAppModel.self) private var appModel + @AppStorage("talk.enabled") private var talkEnabled: Bool = false + @AppStorage("talk.button.enabled") private var talkButtonEnabled: Bool = true + var systemColorScheme: ColorScheme + var gatewayStatus: StatusPill.GatewayState + var voiceWakeEnabled: Bool + var voiceWakeToastText: String? + var cameraHUDText: String? + var cameraHUDKind: NodeAppModel.CameraHUDKind? + var openChat: () -> Void + var openSettings: () -> Void + + private var brightenButtons: Bool { self.systemColorScheme == .light } + + var body: some View { + ZStack(alignment: .topTrailing) { + ScreenTab() + + VStack(spacing: 10) { + OverlayButton(systemImage: "text.bubble.fill", brighten: self.brightenButtons) { + self.openChat() + } + .accessibilityLabel("Chat") + + if self.talkButtonEnabled { + // Talk mode lives on a side bubble so it doesn't get buried in settings. + OverlayButton( + systemImage: self.appModel.talkMode.isEnabled ? "waveform.circle.fill" : "waveform.circle", + brighten: self.brightenButtons, + tint: self.appModel.seamColor, + isActive: self.appModel.talkMode.isEnabled) + { + let next = !self.appModel.talkMode.isEnabled + self.talkEnabled = next + self.appModel.setTalkEnabled(next) + } + .accessibilityLabel("Talk Mode") + } + + OverlayButton(systemImage: "gearshape.fill", brighten: self.brightenButtons) { + self.openSettings() + } + .accessibilityLabel("Settings") + } + .padding(.top, 10) + .padding(.trailing, 10) + } + .overlay(alignment: .center) { + if self.appModel.talkMode.isEnabled { + TalkOrbOverlay() + .transition(.opacity) + } + } + .overlay(alignment: .topLeading) { + StatusPill( + gateway: self.gatewayStatus, + voiceWakeEnabled: self.voiceWakeEnabled, + activity: self.statusActivity, + brighten: self.brightenButtons, + onTap: { + self.openSettings() + }) + .padding(.leading, 10) + .safeAreaPadding(.top, 10) + } + .overlay(alignment: .topLeading) { + if let voiceWakeToastText, !voiceWakeToastText.isEmpty { + VoiceWakeToast( + command: voiceWakeToastText, + brighten: self.brightenButtons) + .padding(.leading, 10) + .safeAreaPadding(.top, 58) + .transition(.move(edge: .top).combined(with: .opacity)) + } + } + } + + private var statusActivity: StatusPill.Activity? { + // Status pill owns transient activity state so it doesn't overlap the connection indicator. + if self.appModel.isBackgrounded { + return StatusPill.Activity( + title: "Foreground required", + systemImage: "exclamationmark.triangle.fill", + tint: .orange) + } + + let gatewayStatus = self.appModel.gatewayStatusText.trimmingCharacters(in: .whitespacesAndNewlines) + let gatewayLower = gatewayStatus.lowercased() + if gatewayLower.contains("repair") { + return StatusPill.Activity(title: "Repairing…", systemImage: "wrench.and.screwdriver", tint: .orange) + } + if gatewayLower.contains("approval") || gatewayLower.contains("pairing") { + return StatusPill.Activity(title: "Approval pending", systemImage: "person.crop.circle.badge.clock") + } + // Avoid duplicating the primary gateway status ("Connecting…") in the activity slot. + + if self.appModel.screenRecordActive { + return StatusPill.Activity(title: "Recording screen…", systemImage: "record.circle.fill", tint: .red) + } + + if let cameraHUDText, !cameraHUDText.isEmpty, let cameraHUDKind { + let systemImage: String + let tint: Color? + switch cameraHUDKind { + case .photo: + systemImage = "camera.fill" + tint = nil + case .recording: + systemImage = "video.fill" + tint = .red + case .success: + systemImage = "checkmark.circle.fill" + tint = .green + case .error: + systemImage = "exclamationmark.triangle.fill" + tint = .red + } + return StatusPill.Activity(title: cameraHUDText, systemImage: systemImage, tint: tint) + } + + if self.voiceWakeEnabled { + let voiceStatus = self.appModel.voiceWake.statusText + if voiceStatus.localizedCaseInsensitiveContains("microphone permission") { + return StatusPill.Activity(title: "Mic permission", systemImage: "mic.slash", tint: .orange) + } + if voiceStatus == "Paused" { + let suffix = self.appModel.isBackgrounded ? " (background)" : "" + return StatusPill.Activity(title: "Voice Wake paused\(suffix)", systemImage: "pause.circle.fill") + } + } + + return nil + } +} + +private struct OverlayButton: View { + let systemImage: String + let brighten: Bool + var tint: Color? + var isActive: Bool = false + let action: () -> Void + + var body: some View { + Button(action: self.action) { + Image(systemName: self.systemImage) + .font(.system(size: 16, weight: .semibold)) + .foregroundStyle(self.isActive ? (self.tint ?? .primary) : .primary) + .padding(10) + .background { + RoundedRectangle(cornerRadius: 12, style: .continuous) + .fill(.ultraThinMaterial) + .overlay { + RoundedRectangle(cornerRadius: 12, style: .continuous) + .fill( + LinearGradient( + colors: [ + .white.opacity(self.brighten ? 0.26 : 0.18), + .white.opacity(self.brighten ? 0.08 : 0.04), + .clear, + ], + startPoint: .topLeading, + endPoint: .bottomTrailing)) + .blendMode(.overlay) + } + .overlay { + if let tint { + RoundedRectangle(cornerRadius: 12, style: .continuous) + .fill( + LinearGradient( + colors: [ + tint.opacity(self.isActive ? 0.22 : 0.14), + tint.opacity(self.isActive ? 0.10 : 0.06), + .clear, + ], + startPoint: .topLeading, + endPoint: .bottomTrailing)) + .blendMode(.overlay) + } + } + .overlay { + RoundedRectangle(cornerRadius: 12, style: .continuous) + .strokeBorder( + (self.tint ?? .white).opacity(self.isActive ? 0.34 : (self.brighten ? 0.24 : 0.18)), + lineWidth: self.isActive ? 0.7 : 0.5) + } + .shadow(color: .black.opacity(0.35), radius: 12, y: 6) + } + } + .buttonStyle(.plain) + } +} + +private struct CameraFlashOverlay: View { + var nonce: Int + + @State private var opacity: CGFloat = 0 + @State private var task: Task? + + var body: some View { + Color.white + .opacity(self.opacity) + .ignoresSafeArea() + .allowsHitTesting(false) + .onChange(of: self.nonce) { _, _ in + self.task?.cancel() + self.task = Task { @MainActor in + withAnimation(.easeOut(duration: 0.08)) { + self.opacity = 0.85 + } + try? await Task.sleep(nanoseconds: 110_000_000) + withAnimation(.easeOut(duration: 0.32)) { + self.opacity = 0 + } + } + } + } +} diff --git a/apps/ios/Sources/RootTabs.swift b/apps/ios/Sources/RootTabs.swift new file mode 100644 index 0000000000000000000000000000000000000000..f7b3fd8226074d0f6587ed9fc2d4005b4fa8f657 --- /dev/null +++ b/apps/ios/Sources/RootTabs.swift @@ -0,0 +1,143 @@ +import SwiftUI + +struct RootTabs: View { + @Environment(NodeAppModel.self) private var appModel + @Environment(VoiceWakeManager.self) private var voiceWake + @AppStorage(VoiceWakePreferences.enabledKey) private var voiceWakeEnabled: Bool = false + @State private var selectedTab: Int = 0 + @State private var voiceWakeToastText: String? + @State private var toastDismissTask: Task? + + var body: some View { + TabView(selection: self.$selectedTab) { + ScreenTab() + .tabItem { Label("Screen", systemImage: "rectangle.and.hand.point.up.left") } + .tag(0) + + VoiceTab() + .tabItem { Label("Voice", systemImage: "mic") } + .tag(1) + + SettingsTab() + .tabItem { Label("Settings", systemImage: "gearshape") } + .tag(2) + } + .overlay(alignment: .topLeading) { + StatusPill( + gateway: self.gatewayStatus, + voiceWakeEnabled: self.voiceWakeEnabled, + activity: self.statusActivity, + onTap: { self.selectedTab = 2 }) + .padding(.leading, 10) + .safeAreaPadding(.top, 10) + } + .overlay(alignment: .topLeading) { + if let voiceWakeToastText, !voiceWakeToastText.isEmpty { + VoiceWakeToast(command: voiceWakeToastText) + .padding(.leading, 10) + .safeAreaPadding(.top, 58) + .transition(.move(edge: .top).combined(with: .opacity)) + } + } + .onChange(of: self.voiceWake.lastTriggeredCommand) { _, newValue in + guard let newValue else { return } + let trimmed = newValue.trimmingCharacters(in: .whitespacesAndNewlines) + guard !trimmed.isEmpty else { return } + + self.toastDismissTask?.cancel() + withAnimation(.spring(response: 0.25, dampingFraction: 0.85)) { + self.voiceWakeToastText = trimmed + } + + self.toastDismissTask = Task { + try? await Task.sleep(nanoseconds: 2_300_000_000) + await MainActor.run { + withAnimation(.easeOut(duration: 0.25)) { + self.voiceWakeToastText = nil + } + } + } + } + .onDisappear { + self.toastDismissTask?.cancel() + self.toastDismissTask = nil + } + } + + private var gatewayStatus: StatusPill.GatewayState { + if self.appModel.gatewayServerName != nil { return .connected } + + let text = self.appModel.gatewayStatusText.trimmingCharacters(in: .whitespacesAndNewlines) + if text.localizedCaseInsensitiveContains("connecting") || + text.localizedCaseInsensitiveContains("reconnecting") + { + return .connecting + } + + if text.localizedCaseInsensitiveContains("error") { + return .error + } + + return .disconnected + } + + private var statusActivity: StatusPill.Activity? { + // Keep the top pill consistent across tabs (camera + voice wake + pairing states). + if self.appModel.isBackgrounded { + return StatusPill.Activity( + title: "Foreground required", + systemImage: "exclamationmark.triangle.fill", + tint: .orange) + } + + let gatewayStatus = self.appModel.gatewayStatusText.trimmingCharacters(in: .whitespacesAndNewlines) + let gatewayLower = gatewayStatus.lowercased() + if gatewayLower.contains("repair") { + return StatusPill.Activity(title: "Repairing…", systemImage: "wrench.and.screwdriver", tint: .orange) + } + if gatewayLower.contains("approval") || gatewayLower.contains("pairing") { + return StatusPill.Activity(title: "Approval pending", systemImage: "person.crop.circle.badge.clock") + } + // Avoid duplicating the primary gateway status ("Connecting…") in the activity slot. + + if self.appModel.screenRecordActive { + return StatusPill.Activity(title: "Recording screen…", systemImage: "record.circle.fill", tint: .red) + } + + if let cameraHUDText = self.appModel.cameraHUDText, + let cameraHUDKind = self.appModel.cameraHUDKind, + !cameraHUDText.isEmpty + { + let systemImage: String + let tint: Color? + switch cameraHUDKind { + case .photo: + systemImage = "camera.fill" + tint = nil + case .recording: + systemImage = "video.fill" + tint = .red + case .success: + systemImage = "checkmark.circle.fill" + tint = .green + case .error: + systemImage = "exclamationmark.triangle.fill" + tint = .red + } + return StatusPill.Activity(title: cameraHUDText, systemImage: systemImage, tint: tint) + } + + if self.voiceWakeEnabled { + let voiceStatus = self.appModel.voiceWake.statusText + if voiceStatus.localizedCaseInsensitiveContains("microphone permission") { + return StatusPill.Activity(title: "Mic permission", systemImage: "mic.slash", tint: .orange) + } + if voiceStatus == "Paused" { + let suffix = self.appModel.isBackgrounded ? " (background)" : "" + return StatusPill.Activity(title: "Voice Wake paused\(suffix)", systemImage: "pause.circle.fill") + } + } + + return nil + } +} diff --git a/apps/ios/Sources/Screen/ScreenController.swift b/apps/ios/Sources/Screen/ScreenController.swift new file mode 100644 index 0000000000000000000000000000000000000000..3fe13a0c9840ce02602c50ac1eec64b539aaf52c --- /dev/null +++ b/apps/ios/Sources/Screen/ScreenController.swift @@ -0,0 +1,411 @@ +import OpenClawKit +import Observation +import SwiftUI +import WebKit + +@MainActor +@Observable +final class ScreenController { + let webView: WKWebView + private let navigationDelegate: ScreenNavigationDelegate + private let a2uiActionHandler: CanvasA2UIActionMessageHandler + + var urlString: String = "" + var errorText: String? + + /// Callback invoked when an openclaw:// deep link is tapped in the canvas + var onDeepLink: ((URL) -> Void)? + + /// Callback invoked when the user clicks an A2UI action (e.g. button) inside the canvas web UI. + var onA2UIAction: (([String: Any]) -> Void)? + + private var debugStatusEnabled: Bool = false + private var debugStatusTitle: String? + private var debugStatusSubtitle: String? + + init() { + let config = WKWebViewConfiguration() + config.websiteDataStore = .nonPersistent() + let a2uiActionHandler = CanvasA2UIActionMessageHandler() + let userContentController = WKUserContentController() + for name in CanvasA2UIActionMessageHandler.handlerNames { + userContentController.add(a2uiActionHandler, name: name) + } + config.userContentController = userContentController + self.navigationDelegate = ScreenNavigationDelegate() + self.a2uiActionHandler = a2uiActionHandler + self.webView = WKWebView(frame: .zero, configuration: config) + // Canvas scaffold is a fully self-contained HTML page; avoid relying on transparency underlays. + self.webView.isOpaque = true + self.webView.backgroundColor = .black + self.webView.scrollView.backgroundColor = .black + self.webView.scrollView.contentInsetAdjustmentBehavior = .never + self.webView.scrollView.contentInset = .zero + self.webView.scrollView.scrollIndicatorInsets = .zero + self.webView.scrollView.automaticallyAdjustsScrollIndicatorInsets = false + self.applyScrollBehavior() + self.webView.navigationDelegate = self.navigationDelegate + self.navigationDelegate.controller = self + a2uiActionHandler.controller = self + self.reload() + } + + func navigate(to urlString: String) { + let trimmed = urlString.trimmingCharacters(in: .whitespacesAndNewlines) + self.urlString = (trimmed == "/" ? "" : trimmed) + self.reload() + } + + func reload() { + let trimmed = self.urlString.trimmingCharacters(in: .whitespacesAndNewlines) + self.applyScrollBehavior() + if trimmed.isEmpty { + guard let url = Self.canvasScaffoldURL else { return } + self.errorText = nil + self.webView.loadFileURL(url, allowingReadAccessTo: url.deletingLastPathComponent()) + return + } else { + guard let url = URL(string: trimmed) else { + self.errorText = "Invalid URL: \(trimmed)" + return + } + self.errorText = nil + if url.isFileURL { + self.webView.loadFileURL(url, allowingReadAccessTo: url.deletingLastPathComponent()) + } else { + self.webView.load(URLRequest(url: url)) + } + } + } + + func showDefaultCanvas() { + self.urlString = "" + self.reload() + } + + func setDebugStatusEnabled(_ enabled: Bool) { + self.debugStatusEnabled = enabled + self.applyDebugStatusIfNeeded() + } + + func updateDebugStatus(title: String?, subtitle: String?) { + self.debugStatusTitle = title + self.debugStatusSubtitle = subtitle + self.applyDebugStatusIfNeeded() + } + + fileprivate func applyDebugStatusIfNeeded() { + let enabled = self.debugStatusEnabled + let title = self.debugStatusTitle + let subtitle = self.debugStatusSubtitle + let js = """ + (() => { + try { + const api = globalThis.__openclaw; + if (!api) return; + if (typeof api.setDebugStatusEnabled === 'function') { + api.setDebugStatusEnabled(\(enabled ? "true" : "false")); + } + if (!\(enabled ? "true" : "false")) return; + if (typeof api.setStatus === 'function') { + api.setStatus(\(Self.jsValue(title)), \(Self.jsValue(subtitle))); + } + } catch (_) {} + })() + """ + self.webView.evaluateJavaScript(js) { _, _ in } + } + + func waitForA2UIReady(timeoutMs: Int) async -> Bool { + let clock = ContinuousClock() + let deadline = clock.now.advanced(by: .milliseconds(timeoutMs)) + while clock.now < deadline { + do { + let res = try await self.eval(javaScript: """ + (() => { + try { + const host = globalThis.openclawA2UI; + return !!host && typeof host.applyMessages === 'function'; + } catch (_) { return false; } + })() + """) + let trimmed = res.trimmingCharacters(in: .whitespacesAndNewlines).lowercased() + if trimmed == "true" || trimmed == "1" { return true } + } catch { + // ignore; page likely still loading + } + try? await Task.sleep(nanoseconds: 120_000_000) + } + return false + } + + func eval(javaScript: String) async throws -> String { + try await withCheckedThrowingContinuation { cont in + self.webView.evaluateJavaScript(javaScript) { result, error in + if let error { + cont.resume(throwing: error) + return + } + if let result { + cont.resume(returning: String(describing: result)) + } else { + cont.resume(returning: "") + } + } + } + } + + func snapshotPNGBase64(maxWidth: CGFloat? = nil) async throws -> String { + let config = WKSnapshotConfiguration() + if let maxWidth { + config.snapshotWidth = NSNumber(value: Double(maxWidth)) + } + let image: UIImage = try await withCheckedThrowingContinuation { cont in + self.webView.takeSnapshot(with: config) { image, error in + if let error { + cont.resume(throwing: error) + return + } + guard let image else { + cont.resume(throwing: NSError(domain: "Screen", code: 2, userInfo: [ + NSLocalizedDescriptionKey: "snapshot failed", + ])) + return + } + cont.resume(returning: image) + } + } + guard let data = image.pngData() else { + throw NSError(domain: "Screen", code: 1, userInfo: [ + NSLocalizedDescriptionKey: "snapshot encode failed", + ]) + } + return data.base64EncodedString() + } + + func snapshotBase64( + maxWidth: CGFloat? = nil, + format: OpenClawCanvasSnapshotFormat, + quality: Double? = nil) async throws -> String + { + let config = WKSnapshotConfiguration() + if let maxWidth { + config.snapshotWidth = NSNumber(value: Double(maxWidth)) + } + let image: UIImage = try await withCheckedThrowingContinuation { cont in + self.webView.takeSnapshot(with: config) { image, error in + if let error { + cont.resume(throwing: error) + return + } + guard let image else { + cont.resume(throwing: NSError(domain: "Screen", code: 2, userInfo: [ + NSLocalizedDescriptionKey: "snapshot failed", + ])) + return + } + cont.resume(returning: image) + } + } + + let data: Data? + switch format { + case .png: + data = image.pngData() + case .jpeg: + let q = (quality ?? 0.82).clamped(to: 0.1...1.0) + data = image.jpegData(compressionQuality: q) + } + guard let data else { + throw NSError(domain: "Screen", code: 1, userInfo: [ + NSLocalizedDescriptionKey: "snapshot encode failed", + ]) + } + return data.base64EncodedString() + } + + private static func bundledResourceURL( + name: String, + ext: String, + subdirectory: String) + -> URL? + { + let bundle = OpenClawKitResources.bundle + return bundle.url(forResource: name, withExtension: ext, subdirectory: subdirectory) + ?? bundle.url(forResource: name, withExtension: ext) + } + + private static let canvasScaffoldURL: URL? = ScreenController.bundledResourceURL( + name: "scaffold", + ext: "html", + subdirectory: "CanvasScaffold") + func isTrustedCanvasUIURL(_ url: URL) -> Bool { + guard url.isFileURL else { return false } + let std = url.standardizedFileURL + if let expected = Self.canvasScaffoldURL, + std == expected.standardizedFileURL + { + return true + } + return false + } + + private func applyScrollBehavior() { + let trimmed = self.urlString.trimmingCharacters(in: .whitespacesAndNewlines) + let allowScroll = !trimmed.isEmpty + let scrollView = self.webView.scrollView + // Default canvas needs raw touch events; external pages should scroll. + scrollView.isScrollEnabled = allowScroll + scrollView.bounces = allowScroll + } + + private static func jsValue(_ value: String?) -> String { + guard let value else { return "null" } + if let data = try? JSONSerialization.data(withJSONObject: [value]), + let encoded = String(data: data, encoding: .utf8), + encoded.count >= 2 + { + return String(encoded.dropFirst().dropLast()) + } + return "null" + } + + func isLocalNetworkCanvasURL(_ url: URL) -> Bool { + guard let scheme = url.scheme?.lowercased(), scheme == "http" || scheme == "https" else { + return false + } + guard let host = url.host?.trimmingCharacters(in: .whitespacesAndNewlines), !host.isEmpty else { + return false + } + if host == "localhost" { return true } + if host.hasSuffix(".local") { return true } + if host.hasSuffix(".ts.net") { return true } + if host.hasSuffix(".tailscale.net") { return true } + // Allow MagicDNS / LAN hostnames like "peters-mac-studio-1". + if !host.contains("."), !host.contains(":") { return true } + if let ipv4 = Self.parseIPv4(host) { + return Self.isLocalNetworkIPv4(ipv4) + } + return false + } + + private static func parseIPv4(_ host: String) -> (UInt8, UInt8, UInt8, UInt8)? { + let parts = host.split(separator: ".", omittingEmptySubsequences: false) + guard parts.count == 4 else { return nil } + let bytes: [UInt8] = parts.compactMap { UInt8($0) } + guard bytes.count == 4 else { return nil } + return (bytes[0], bytes[1], bytes[2], bytes[3]) + } + + private static func isLocalNetworkIPv4(_ ip: (UInt8, UInt8, UInt8, UInt8)) -> Bool { + let (a, b, _, _) = ip + // 10.0.0.0/8 + if a == 10 { return true } + // 172.16.0.0/12 + if a == 172, (16...31).contains(Int(b)) { return true } + // 192.168.0.0/16 + if a == 192, b == 168 { return true } + // 127.0.0.0/8 + if a == 127 { return true } + // 169.254.0.0/16 (link-local) + if a == 169, b == 254 { return true } + // Tailscale: 100.64.0.0/10 + if a == 100, (64...127).contains(Int(b)) { return true } + return false + } + + nonisolated static func parseA2UIActionBody(_ body: Any) -> [String: Any]? { + if let dict = body as? [String: Any] { return dict.isEmpty ? nil : dict } + if let str = body as? String, + let data = str.data(using: .utf8), + let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any] + { + return json.isEmpty ? nil : json + } + if let dict = body as? [AnyHashable: Any] { + let mapped = dict.reduce(into: [String: Any]()) { acc, pair in + guard let key = pair.key as? String else { return } + acc[key] = pair.value + } + return mapped.isEmpty ? nil : mapped + } + return nil + } +} + +extension Double { + fileprivate func clamped(to range: ClosedRange) -> Double { + if self < range.lowerBound { return range.lowerBound } + if self > range.upperBound { return range.upperBound } + return self + } +} + +// MARK: - Navigation Delegate + +/// Handles navigation policy to intercept openclaw:// deep links from canvas +@MainActor +private final class ScreenNavigationDelegate: NSObject, WKNavigationDelegate { + weak var controller: ScreenController? + + func webView( + _ webView: WKWebView, + decidePolicyFor navigationAction: WKNavigationAction, + decisionHandler: @escaping @MainActor @Sendable (WKNavigationActionPolicy) -> Void) + { + guard let url = navigationAction.request.url else { + decisionHandler(.allow) + return + } + + // Intercept openclaw:// deep links. + if url.scheme?.lowercased() == "openclaw" { + decisionHandler(.cancel) + self.controller?.onDeepLink?(url) + return + } + + decisionHandler(.allow) + } + + func webView( + _: WKWebView, + didFailProvisionalNavigation _: WKNavigation?, + withError error: any Error) + { + self.controller?.errorText = error.localizedDescription + } + + func webView(_: WKWebView, didFinish _: WKNavigation?) { + self.controller?.errorText = nil + self.controller?.applyDebugStatusIfNeeded() + } + + func webView(_: WKWebView, didFail _: WKNavigation?, withError error: any Error) { + self.controller?.errorText = error.localizedDescription + } +} + +private final class CanvasA2UIActionMessageHandler: NSObject, WKScriptMessageHandler { + static let messageName = "openclawCanvasA2UIAction" + static let handlerNames = [messageName] + + weak var controller: ScreenController? + + func userContentController(_: WKUserContentController, didReceive message: WKScriptMessage) { + guard Self.handlerNames.contains(message.name) else { return } + guard let controller else { return } + + guard let url = message.webView?.url else { return } + if url.isFileURL { + guard controller.isTrustedCanvasUIURL(url) else { return } + } else { + // For security, only accept actions from local-network pages (e.g. the canvas host). + guard controller.isLocalNetworkCanvasURL(url) else { return } + } + + guard let body = ScreenController.parseA2UIActionBody(message.body) else { return } + + controller.onA2UIAction?(body) + } +} diff --git a/apps/ios/Sources/Screen/ScreenRecordService.swift b/apps/ios/Sources/Screen/ScreenRecordService.swift new file mode 100644 index 0000000000000000000000000000000000000000..11052f2354325ccf71cb7e78df2454468829dcf9 --- /dev/null +++ b/apps/ios/Sources/Screen/ScreenRecordService.swift @@ -0,0 +1,360 @@ +import AVFoundation +import ReplayKit + +final class ScreenRecordService: @unchecked Sendable { + private struct UncheckedSendableBox: @unchecked Sendable { + let value: T + } + + private final class CaptureState: @unchecked Sendable { + private let lock = NSLock() + var writer: AVAssetWriter? + var videoInput: AVAssetWriterInput? + var audioInput: AVAssetWriterInput? + var started = false + var sawVideo = false + var lastVideoTime: CMTime? + var handlerError: Error? + + func withLock(_ body: (CaptureState) -> T) -> T { + self.lock.lock() + defer { lock.unlock() } + return body(self) + } + } + + enum ScreenRecordError: LocalizedError { + case invalidScreenIndex(Int) + case captureFailed(String) + case writeFailed(String) + + var errorDescription: String? { + switch self { + case let .invalidScreenIndex(idx): + "Invalid screen index \(idx)" + case let .captureFailed(msg): + msg + case let .writeFailed(msg): + msg + } + } + } + + func record( + screenIndex: Int?, + durationMs: Int?, + fps: Double?, + includeAudio: Bool?, + outPath: String?) async throws -> String + { + let config = try self.makeRecordConfig( + screenIndex: screenIndex, + durationMs: durationMs, + fps: fps, + includeAudio: includeAudio, + outPath: outPath) + + let state = CaptureState() + let recordQueue = DispatchQueue(label: "bot.molt.screenrecord") + + try await self.startCapture(state: state, config: config, recordQueue: recordQueue) + try await Task.sleep(nanoseconds: UInt64(config.durationMs) * 1_000_000) + try await self.stopCapture() + try self.finalizeCapture(state: state) + try await self.finishWriting(state: state) + + return config.outURL.path + } + + private struct RecordConfig { + let durationMs: Int + let fpsValue: Double + let includeAudio: Bool + let outURL: URL + } + + private func makeRecordConfig( + screenIndex: Int?, + durationMs: Int?, + fps: Double?, + includeAudio: Bool?, + outPath: String?) throws -> RecordConfig + { + if let idx = screenIndex, idx != 0 { + throw ScreenRecordError.invalidScreenIndex(idx) + } + + let durationMs = Self.clampDurationMs(durationMs) + let fps = Self.clampFps(fps) + let fpsInt = Int32(fps.rounded()) + let fpsValue = Double(fpsInt) + let includeAudio = includeAudio ?? true + + let outURL = self.makeOutputURL(outPath: outPath) + try? FileManager().removeItem(at: outURL) + + return RecordConfig( + durationMs: durationMs, + fpsValue: fpsValue, + includeAudio: includeAudio, + outURL: outURL) + } + + private func makeOutputURL(outPath: String?) -> URL { + if let outPath, !outPath.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty { + return URL(fileURLWithPath: outPath) + } + return FileManager().temporaryDirectory + .appendingPathComponent("openclaw-screen-record-\(UUID().uuidString).mp4") + } + + private func startCapture( + state: CaptureState, + config: RecordConfig, + recordQueue: DispatchQueue) async throws + { + try await withCheckedThrowingContinuation { (cont: CheckedContinuation) in + let handler = self.makeCaptureHandler( + state: state, + config: config, + recordQueue: recordQueue) + let completion: @Sendable (Error?) -> Void = { error in + if let error { cont.resume(throwing: error) } else { cont.resume() } + } + + Task { @MainActor in + startReplayKitCapture( + includeAudio: config.includeAudio, + handler: handler, + completion: completion) + } + } + } + + private func makeCaptureHandler( + state: CaptureState, + config: RecordConfig, + recordQueue: DispatchQueue) -> @Sendable (CMSampleBuffer, RPSampleBufferType, Error?) -> Void + { + { sample, type, error in + let sampleBox = UncheckedSendableBox(value: sample) + // ReplayKit can call the capture handler on a background queue. + // Serialize writes to avoid queue asserts. + recordQueue.async { + let sample = sampleBox.value + if let error { + state.withLock { state in + if state.handlerError == nil { state.handlerError = error } + } + return + } + guard CMSampleBufferDataIsReady(sample) else { return } + + switch type { + case .video: + self.handleVideoSample(sample, state: state, config: config) + case .audioApp, .audioMic: + self.handleAudioSample(sample, state: state, includeAudio: config.includeAudio) + @unknown default: + break + } + } + } + } + + private func handleVideoSample( + _ sample: CMSampleBuffer, + state: CaptureState, + config: RecordConfig) + { + let pts = CMSampleBufferGetPresentationTimeStamp(sample) + let shouldSkip = state.withLock { state in + if let lastVideoTime = state.lastVideoTime { + let delta = CMTimeSubtract(pts, lastVideoTime) + return delta.seconds < (1.0 / config.fpsValue) + } + return false + } + if shouldSkip { return } + + if state.withLock({ $0.writer == nil }) { + self.prepareWriter(sample: sample, state: state, config: config, pts: pts) + } + + let vInput = state.withLock { $0.videoInput } + let isStarted = state.withLock { $0.started } + guard let vInput, isStarted else { return } + if vInput.isReadyForMoreMediaData { + if vInput.append(sample) { + state.withLock { state in + state.sawVideo = true + state.lastVideoTime = pts + } + } else { + let err = state.withLock { $0.writer?.error } + if let err { + state.withLock { state in + if state.handlerError == nil { + state.handlerError = ScreenRecordError.writeFailed(err.localizedDescription) + } + } + } + } + } + } + + private func prepareWriter( + sample: CMSampleBuffer, + state: CaptureState, + config: RecordConfig, + pts: CMTime) + { + guard let imageBuffer = CMSampleBufferGetImageBuffer(sample) else { + state.withLock { state in + if state.handlerError == nil { + state.handlerError = ScreenRecordError.captureFailed("Missing image buffer") + } + } + return + } + let width = CVPixelBufferGetWidth(imageBuffer) + let height = CVPixelBufferGetHeight(imageBuffer) + do { + let writer = try AVAssetWriter(outputURL: config.outURL, fileType: .mp4) + let settings: [String: Any] = [ + AVVideoCodecKey: AVVideoCodecType.h264, + AVVideoWidthKey: width, + AVVideoHeightKey: height, + ] + let vInput = AVAssetWriterInput(mediaType: .video, outputSettings: settings) + vInput.expectsMediaDataInRealTime = true + guard writer.canAdd(vInput) else { + throw ScreenRecordError.writeFailed("Cannot add video input") + } + writer.add(vInput) + + if config.includeAudio { + let aInput = AVAssetWriterInput(mediaType: .audio, outputSettings: nil) + aInput.expectsMediaDataInRealTime = true + if writer.canAdd(aInput) { + writer.add(aInput) + state.withLock { state in + state.audioInput = aInput + } + } + } + + guard writer.startWriting() else { + throw ScreenRecordError.writeFailed( + writer.error?.localizedDescription ?? "Failed to start writer") + } + writer.startSession(atSourceTime: pts) + state.withLock { state in + state.writer = writer + state.videoInput = vInput + state.started = true + } + } catch { + state.withLock { state in + if state.handlerError == nil { state.handlerError = error } + } + } + } + + private func handleAudioSample( + _ sample: CMSampleBuffer, + state: CaptureState, + includeAudio: Bool) + { + let aInput = state.withLock { $0.audioInput } + let isStarted = state.withLock { $0.started } + guard includeAudio, let aInput, isStarted else { return } + if aInput.isReadyForMoreMediaData { + _ = aInput.append(sample) + } + } + + private func stopCapture() async throws { + let stopError = await withCheckedContinuation { cont in + Task { @MainActor in + stopReplayKitCapture { error in cont.resume(returning: error) } + } + } + if let stopError { throw stopError } + } + + private func finalizeCapture(state: CaptureState) throws { + if let handlerErrorSnapshot = state.withLock({ $0.handlerError }) { + throw handlerErrorSnapshot + } + let writerSnapshot = state.withLock { $0.writer } + let videoInputSnapshot = state.withLock { $0.videoInput } + let audioInputSnapshot = state.withLock { $0.audioInput } + let sawVideoSnapshot = state.withLock { $0.sawVideo } + guard let writerSnapshot, let videoInputSnapshot, sawVideoSnapshot else { + throw ScreenRecordError.captureFailed("No frames captured") + } + + videoInputSnapshot.markAsFinished() + audioInputSnapshot?.markAsFinished() + _ = writerSnapshot + } + + private func finishWriting(state: CaptureState) async throws { + guard let writerSnapshot = state.withLock({ $0.writer }) else { + throw ScreenRecordError.captureFailed("Missing writer") + } + let writerBox = UncheckedSendableBox(value: writerSnapshot) + try await withCheckedThrowingContinuation { (cont: CheckedContinuation) in + writerBox.value.finishWriting { + let writer = writerBox.value + if let err = writer.error { + cont.resume(throwing: ScreenRecordError.writeFailed(err.localizedDescription)) + } else if writer.status != .completed { + cont.resume(throwing: ScreenRecordError.writeFailed("Failed to finalize video")) + } else { + cont.resume() + } + } + } + } + + private nonisolated static func clampDurationMs(_ ms: Int?) -> Int { + let v = ms ?? 10000 + return min(60000, max(250, v)) + } + + private nonisolated static func clampFps(_ fps: Double?) -> Double { + let v = fps ?? 10 + if !v.isFinite { return 10 } + return min(30, max(1, v)) + } +} + +@MainActor +private func startReplayKitCapture( + includeAudio: Bool, + handler: @escaping @Sendable (CMSampleBuffer, RPSampleBufferType, Error?) -> Void, + completion: @escaping @Sendable (Error?) -> Void) +{ + let recorder = RPScreenRecorder.shared() + recorder.isMicrophoneEnabled = includeAudio + recorder.startCapture(handler: handler, completionHandler: completion) +} + +@MainActor +private func stopReplayKitCapture(_ completion: @escaping @Sendable (Error?) -> Void) { + RPScreenRecorder.shared().stopCapture { error in completion(error) } +} + +#if DEBUG +extension ScreenRecordService { + nonisolated static func _test_clampDurationMs(_ ms: Int?) -> Int { + self.clampDurationMs(ms) + } + + nonisolated static func _test_clampFps(_ fps: Double?) -> Double { + self.clampFps(fps) + } +} +#endif diff --git a/apps/ios/Sources/Screen/ScreenTab.swift b/apps/ios/Sources/Screen/ScreenTab.swift new file mode 100644 index 0000000000000000000000000000000000000000..fd3d0276d395395d84cf2442fb12bdb585975d0b --- /dev/null +++ b/apps/ios/Sources/Screen/ScreenTab.swift @@ -0,0 +1,25 @@ +import OpenClawKit +import SwiftUI + +struct ScreenTab: View { + @Environment(NodeAppModel.self) private var appModel + + var body: some View { + ZStack(alignment: .top) { + ScreenWebView(controller: self.appModel.screen) + .ignoresSafeArea() + .overlay(alignment: .top) { + if let errorText = self.appModel.screen.errorText { + Text(errorText) + .font(.footnote) + .padding(10) + .background(.thinMaterial) + .clipShape(RoundedRectangle(cornerRadius: 12, style: .continuous)) + .padding() + } + } + } + } + + // Navigation is agent-driven; no local URL bar here. +} diff --git a/apps/ios/Sources/Screen/ScreenWebView.swift b/apps/ios/Sources/Screen/ScreenWebView.swift new file mode 100644 index 0000000000000000000000000000000000000000..c464521be5f9f060dae83178458a455f03e54a40 --- /dev/null +++ b/apps/ios/Sources/Screen/ScreenWebView.swift @@ -0,0 +1,15 @@ +import OpenClawKit +import SwiftUI +import WebKit + +struct ScreenWebView: UIViewRepresentable { + var controller: ScreenController + + func makeUIView(context: Context) -> WKWebView { + self.controller.webView + } + + func updateUIView(_ webView: WKWebView, context: Context) { + // State changes are driven by ScreenController. + } +} diff --git a/apps/ios/Sources/SessionKey.swift b/apps/ios/Sources/SessionKey.swift new file mode 100644 index 0000000000000000000000000000000000000000..bac73f670d33828eeb5f8fc47e139d424bc19693 --- /dev/null +++ b/apps/ios/Sources/SessionKey.swift @@ -0,0 +1,15 @@ +import Foundation + +enum SessionKey { + static func normalizeMainKey(_ raw: String?) -> String { + let trimmed = (raw ?? "").trimmingCharacters(in: .whitespacesAndNewlines) + return trimmed.isEmpty ? "main" : trimmed + } + + static func isCanonicalMainSessionKey(_ value: String?) -> Bool { + let trimmed = (value ?? "").trimmingCharacters(in: .whitespacesAndNewlines) + if trimmed.isEmpty { return false } + if trimmed == "global" { return true } + return trimmed.hasPrefix("agent:") + } +} diff --git a/apps/ios/Sources/Settings/SettingsNetworkingHelpers.swift b/apps/ios/Sources/Settings/SettingsNetworkingHelpers.swift new file mode 100644 index 0000000000000000000000000000000000000000..f061ff9a2045d6a19aaad4e53be9c995737e7c91 --- /dev/null +++ b/apps/ios/Sources/Settings/SettingsNetworkingHelpers.swift @@ -0,0 +1,40 @@ +import Foundation + +struct SettingsHostPort: Equatable { + var host: String + var port: Int +} + +enum SettingsNetworkingHelpers { + static func parseHostPort(from address: String) -> SettingsHostPort? { + let trimmed = address.trimmingCharacters(in: .whitespacesAndNewlines) + guard !trimmed.isEmpty else { return nil } + + if trimmed.hasPrefix("["), + let close = trimmed.firstIndex(of: "]"), + close < trimmed.endIndex + { + let host = String(trimmed[trimmed.index(after: trimmed.startIndex).. String { + if let host, let port { + let needsBrackets = host.contains(":") && !host.hasPrefix("[") && !host.hasSuffix("]") + let hostPart = needsBrackets ? "[\(host)]" : host + return "http://\(hostPart):\(port)" + } + return "http://\(fallback)" + } +} diff --git a/apps/ios/Sources/Settings/SettingsTab.swift b/apps/ios/Sources/Settings/SettingsTab.swift new file mode 100644 index 0000000000000000000000000000000000000000..c1ee6099480a1199bca9d845fcd171c248e45134 --- /dev/null +++ b/apps/ios/Sources/Settings/SettingsTab.swift @@ -0,0 +1,466 @@ +import OpenClawKit +import Network +import Observation +import SwiftUI +import UIKit + +@MainActor +@Observable +private final class ConnectStatusStore { + var text: String? +} + +extension ConnectStatusStore: @unchecked Sendable {} + +struct SettingsTab: View { + @Environment(NodeAppModel.self) private var appModel: NodeAppModel + @Environment(VoiceWakeManager.self) private var voiceWake: VoiceWakeManager + @Environment(GatewayConnectionController.self) private var gatewayController: GatewayConnectionController + @Environment(\.dismiss) private var dismiss + @AppStorage("node.displayName") private var displayName: String = "iOS Node" + @AppStorage("node.instanceId") private var instanceId: String = UUID().uuidString + @AppStorage("voiceWake.enabled") private var voiceWakeEnabled: Bool = false + @AppStorage("talk.enabled") private var talkEnabled: Bool = false + @AppStorage("talk.button.enabled") private var talkButtonEnabled: Bool = true + @AppStorage("camera.enabled") private var cameraEnabled: Bool = true + @AppStorage("location.enabledMode") private var locationEnabledModeRaw: String = OpenClawLocationMode.off.rawValue + @AppStorage("location.preciseEnabled") private var locationPreciseEnabled: Bool = true + @AppStorage("screen.preventSleep") private var preventSleep: Bool = true + @AppStorage("gateway.preferredStableID") private var preferredGatewayStableID: String = "" + @AppStorage("gateway.lastDiscoveredStableID") private var lastDiscoveredGatewayStableID: String = "" + @AppStorage("gateway.manual.enabled") private var manualGatewayEnabled: Bool = false + @AppStorage("gateway.manual.host") private var manualGatewayHost: String = "" + @AppStorage("gateway.manual.port") private var manualGatewayPort: Int = 18789 + @AppStorage("gateway.manual.tls") private var manualGatewayTLS: Bool = true + @AppStorage("gateway.discovery.debugLogs") private var discoveryDebugLogsEnabled: Bool = false + @AppStorage("canvas.debugStatusEnabled") private var canvasDebugStatusEnabled: Bool = false + @State private var connectStatus = ConnectStatusStore() + @State private var connectingGatewayID: String? + @State private var localIPAddress: String? + @State private var lastLocationModeRaw: String = OpenClawLocationMode.off.rawValue + @State private var gatewayToken: String = "" + @State private var gatewayPassword: String = "" + + var body: some View { + NavigationStack { + Form { + Section("Node") { + TextField("Name", text: self.$displayName) + Text(self.instanceId) + .font(.footnote) + .foregroundStyle(.secondary) + LabeledContent("IP", value: self.localIPAddress ?? "—") + .contextMenu { + if let ip = self.localIPAddress { + Button { + UIPasteboard.general.string = ip + } label: { + Label("Copy", systemImage: "doc.on.doc") + } + } + } + LabeledContent("Platform", value: self.platformString()) + LabeledContent("Version", value: self.appVersion()) + LabeledContent("Model", value: self.modelIdentifier()) + } + + Section("Gateway") { + LabeledContent("Discovery", value: self.gatewayController.discoveryStatusText) + LabeledContent("Status", value: self.appModel.gatewayStatusText) + if let serverName = self.appModel.gatewayServerName { + LabeledContent("Server", value: serverName) + if let addr = self.appModel.gatewayRemoteAddress { + let parts = Self.parseHostPort(from: addr) + let urlString = Self.httpURLString(host: parts?.host, port: parts?.port, fallback: addr) + LabeledContent("Address") { + Text(urlString) + } + .contextMenu { + Button { + UIPasteboard.general.string = urlString + } label: { + Label("Copy URL", systemImage: "doc.on.doc") + } + + if let parts { + Button { + UIPasteboard.general.string = parts.host + } label: { + Label("Copy Host", systemImage: "doc.on.doc") + } + + Button { + UIPasteboard.general.string = "\(parts.port)" + } label: { + Label("Copy Port", systemImage: "doc.on.doc") + } + } + } + } + + Button("Disconnect", role: .destructive) { + self.appModel.disconnectGateway() + } + + self.gatewayList(showing: .availableOnly) + } else { + self.gatewayList(showing: .all) + } + + if let text = self.connectStatus.text { + Text(text) + .font(.footnote) + .foregroundStyle(.secondary) + } + + DisclosureGroup("Advanced") { + Toggle("Use Manual Gateway", isOn: self.$manualGatewayEnabled) + + TextField("Host", text: self.$manualGatewayHost) + .textInputAutocapitalization(.never) + .autocorrectionDisabled() + + TextField("Port", value: self.$manualGatewayPort, format: .number) + .keyboardType(.numberPad) + + Toggle("Use TLS", isOn: self.$manualGatewayTLS) + + Button { + Task { await self.connectManual() } + } label: { + if self.connectingGatewayID == "manual" { + HStack(spacing: 8) { + ProgressView() + .progressViewStyle(.circular) + Text("Connecting…") + } + } else { + Text("Connect (Manual)") + } + } + .disabled(self.connectingGatewayID != nil || self.manualGatewayHost + .trimmingCharacters(in: .whitespacesAndNewlines) + .isEmpty || self.manualGatewayPort <= 0 || self.manualGatewayPort > 65535) + + Text( + "Use this when mDNS/Bonjour discovery is blocked. " + + "The gateway WebSocket listens on port 18789 by default.") + .font(.footnote) + .foregroundStyle(.secondary) + + Toggle("Discovery Debug Logs", isOn: self.$discoveryDebugLogsEnabled) + .onChange(of: self.discoveryDebugLogsEnabled) { _, newValue in + self.gatewayController.setDiscoveryDebugLoggingEnabled(newValue) + } + + NavigationLink("Discovery Logs") { + GatewayDiscoveryDebugLogView() + } + + Toggle("Debug Canvas Status", isOn: self.$canvasDebugStatusEnabled) + + TextField("Gateway Token", text: self.$gatewayToken) + .textInputAutocapitalization(.never) + .autocorrectionDisabled() + + SecureField("Gateway Password", text: self.$gatewayPassword) + } + } + + Section("Voice") { + Toggle("Voice Wake", isOn: self.$voiceWakeEnabled) + .onChange(of: self.voiceWakeEnabled) { _, newValue in + self.appModel.setVoiceWakeEnabled(newValue) + } + Toggle("Talk Mode", isOn: self.$talkEnabled) + .onChange(of: self.talkEnabled) { _, newValue in + self.appModel.setTalkEnabled(newValue) + } + // Keep this separate so users can hide the side bubble without disabling Talk Mode. + Toggle("Show Talk Button", isOn: self.$talkButtonEnabled) + + NavigationLink { + VoiceWakeWordsSettingsView() + } label: { + LabeledContent( + "Wake Words", + value: VoiceWakePreferences.displayString(for: self.voiceWake.triggerWords)) + } + } + + Section("Camera") { + Toggle("Allow Camera", isOn: self.$cameraEnabled) + Text("Allows the gateway to request photos or short video clips (foreground only).") + .font(.footnote) + .foregroundStyle(.secondary) + } + + Section("Location") { + Picker("Location Access", selection: self.$locationEnabledModeRaw) { + Text("Off").tag(OpenClawLocationMode.off.rawValue) + Text("While Using").tag(OpenClawLocationMode.whileUsing.rawValue) + Text("Always").tag(OpenClawLocationMode.always.rawValue) + } + .pickerStyle(.segmented) + + Toggle("Precise Location", isOn: self.$locationPreciseEnabled) + .disabled(self.locationMode == .off) + + Text("Always requires system permission and may prompt to open Settings.") + .font(.footnote) + .foregroundStyle(.secondary) + } + + Section("Screen") { + Toggle("Prevent Sleep", isOn: self.$preventSleep) + Text("Keeps the screen awake while OpenClaw is open.") + .font(.footnote) + .foregroundStyle(.secondary) + } + } + .navigationTitle("Settings") + .toolbar { + ToolbarItem(placement: .topBarTrailing) { + Button { + self.dismiss() + } label: { + Image(systemName: "xmark") + } + .accessibilityLabel("Close") + } + } + .onAppear { + self.localIPAddress = Self.primaryIPv4Address() + self.lastLocationModeRaw = self.locationEnabledModeRaw + let trimmedInstanceId = self.instanceId.trimmingCharacters(in: .whitespacesAndNewlines) + if !trimmedInstanceId.isEmpty { + self.gatewayToken = GatewaySettingsStore.loadGatewayToken(instanceId: trimmedInstanceId) ?? "" + self.gatewayPassword = GatewaySettingsStore.loadGatewayPassword(instanceId: trimmedInstanceId) ?? "" + } + } + .onChange(of: self.preferredGatewayStableID) { _, newValue in + let trimmed = newValue.trimmingCharacters(in: .whitespacesAndNewlines) + guard !trimmed.isEmpty else { return } + GatewaySettingsStore.savePreferredGatewayStableID(trimmed) + } + .onChange(of: self.gatewayToken) { _, newValue in + let trimmed = newValue.trimmingCharacters(in: .whitespacesAndNewlines) + let instanceId = self.instanceId.trimmingCharacters(in: .whitespacesAndNewlines) + guard !instanceId.isEmpty else { return } + GatewaySettingsStore.saveGatewayToken(trimmed, instanceId: instanceId) + } + .onChange(of: self.gatewayPassword) { _, newValue in + let trimmed = newValue.trimmingCharacters(in: .whitespacesAndNewlines) + let instanceId = self.instanceId.trimmingCharacters(in: .whitespacesAndNewlines) + guard !instanceId.isEmpty else { return } + GatewaySettingsStore.saveGatewayPassword(trimmed, instanceId: instanceId) + } + .onChange(of: self.appModel.gatewayServerName) { _, _ in + self.connectStatus.text = nil + } + .onChange(of: self.locationEnabledModeRaw) { _, newValue in + let previous = self.lastLocationModeRaw + self.lastLocationModeRaw = newValue + guard let mode = OpenClawLocationMode(rawValue: newValue) else { return } + Task { + let granted = await self.appModel.requestLocationPermissions(mode: mode) + if !granted { + await MainActor.run { + self.locationEnabledModeRaw = previous + self.lastLocationModeRaw = previous + } + } + } + } + } + } + + @ViewBuilder + private func gatewayList(showing: GatewayListMode) -> some View { + if self.gatewayController.gateways.isEmpty { + Text("No gateways found yet.") + .foregroundStyle(.secondary) + } else { + let connectedID = self.appModel.connectedGatewayID + let rows = self.gatewayController.gateways.filter { gateway in + let isConnected = gateway.stableID == connectedID + switch showing { + case .all: + return true + case .availableOnly: + return !isConnected + } + } + + if rows.isEmpty, showing == .availableOnly { + Text("No other gateways found.") + .foregroundStyle(.secondary) + } else { + ForEach(rows) { gateway in + HStack { + VStack(alignment: .leading, spacing: 2) { + Text(gateway.name) + let detailLines = self.gatewayDetailLines(gateway) + ForEach(detailLines, id: \.self) { line in + Text(line) + .font(.footnote) + .foregroundStyle(.secondary) + } + } + Spacer() + + Button { + Task { await self.connect(gateway) } + } label: { + if self.connectingGatewayID == gateway.id { + ProgressView() + .progressViewStyle(.circular) + } else { + Text("Connect") + } + } + .disabled(self.connectingGatewayID != nil) + } + } + } + } + } + + private enum GatewayListMode: Equatable { + case all + case availableOnly + } + + private func platformString() -> String { + let v = ProcessInfo.processInfo.operatingSystemVersion + return "iOS \(v.majorVersion).\(v.minorVersion).\(v.patchVersion)" + } + + private var locationMode: OpenClawLocationMode { + OpenClawLocationMode(rawValue: self.locationEnabledModeRaw) ?? .off + } + + private func appVersion() -> String { + Bundle.main.infoDictionary?["CFBundleShortVersionString"] as? String ?? "dev" + } + + private func deviceFamily() -> String { + switch UIDevice.current.userInterfaceIdiom { + case .pad: + "iPad" + case .phone: + "iPhone" + default: + "iOS" + } + } + + private func modelIdentifier() -> String { + var systemInfo = utsname() + uname(&systemInfo) + let machine = withUnsafeBytes(of: &systemInfo.machine) { ptr in + String(bytes: ptr.prefix { $0 != 0 }, encoding: .utf8) + } + let trimmed = machine?.trimmingCharacters(in: .whitespacesAndNewlines) ?? "" + return trimmed.isEmpty ? "unknown" : trimmed + } + + private func connect(_ gateway: GatewayDiscoveryModel.DiscoveredGateway) async { + self.connectingGatewayID = gateway.id + self.manualGatewayEnabled = false + self.preferredGatewayStableID = gateway.stableID + GatewaySettingsStore.savePreferredGatewayStableID(gateway.stableID) + self.lastDiscoveredGatewayStableID = gateway.stableID + GatewaySettingsStore.saveLastDiscoveredGatewayStableID(gateway.stableID) + defer { self.connectingGatewayID = nil } + + await self.gatewayController.connect(gateway) + } + + private func connectManual() async { + let host = self.manualGatewayHost.trimmingCharacters(in: .whitespacesAndNewlines) + guard !host.isEmpty else { + self.connectStatus.text = "Failed: host required" + return + } + guard self.manualGatewayPort > 0, self.manualGatewayPort <= 65535 else { + self.connectStatus.text = "Failed: invalid port" + return + } + + self.connectingGatewayID = "manual" + self.manualGatewayEnabled = true + defer { self.connectingGatewayID = nil } + + await self.gatewayController.connectManual( + host: host, + port: self.manualGatewayPort, + useTLS: self.manualGatewayTLS) + } + + private static func primaryIPv4Address() -> String? { + var addrList: UnsafeMutablePointer? + guard getifaddrs(&addrList) == 0, let first = addrList else { return nil } + defer { freeifaddrs(addrList) } + + var fallback: String? + var en0: String? + + for ptr in sequence(first: first, next: { $0.pointee.ifa_next }) { + let flags = Int32(ptr.pointee.ifa_flags) + let isUp = (flags & IFF_UP) != 0 + let isLoopback = (flags & IFF_LOOPBACK) != 0 + let name = String(cString: ptr.pointee.ifa_name) + let family = ptr.pointee.ifa_addr.pointee.sa_family + if !isUp || isLoopback || family != UInt8(AF_INET) { continue } + + var addr = ptr.pointee.ifa_addr.pointee + var buffer = [CChar](repeating: 0, count: Int(NI_MAXHOST)) + let result = getnameinfo( + &addr, + socklen_t(ptr.pointee.ifa_addr.pointee.sa_len), + &buffer, + socklen_t(buffer.count), + nil, + 0, + NI_NUMERICHOST) + guard result == 0 else { continue } + let len = buffer.prefix { $0 != 0 } + let bytes = len.map { UInt8(bitPattern: $0) } + guard let ip = String(bytes: bytes, encoding: .utf8) else { continue } + + if name == "en0" { en0 = ip; break } + if fallback == nil { fallback = ip } + } + + return en0 ?? fallback + } + + private static func parseHostPort(from address: String) -> SettingsHostPort? { + SettingsNetworkingHelpers.parseHostPort(from: address) + } + + private static func httpURLString(host: String?, port: Int?, fallback: String) -> String { + SettingsNetworkingHelpers.httpURLString(host: host, port: port, fallback: fallback) + } + + private func gatewayDetailLines(_ gateway: GatewayDiscoveryModel.DiscoveredGateway) -> [String] { + var lines: [String] = [] + if let lanHost = gateway.lanHost { lines.append("LAN: \(lanHost)") } + if let tailnet = gateway.tailnetDns { lines.append("Tailnet: \(tailnet)") } + + let gatewayPort = gateway.gatewayPort + let canvasPort = gateway.canvasPort + if gatewayPort != nil || canvasPort != nil { + let gw = gatewayPort.map(String.init) ?? "—" + let canvas = canvasPort.map(String.init) ?? "—" + lines.append("Ports: gateway \(gw) · canvas \(canvas)") + } + + if lines.isEmpty { + lines.append(gateway.debugID) + } + + return lines + } +} diff --git a/apps/ios/Sources/Settings/VoiceWakeWordsSettingsView.swift b/apps/ios/Sources/Settings/VoiceWakeWordsSettingsView.swift new file mode 100644 index 0000000000000000000000000000000000000000..e00e87e55d688362284d66a08b25ad04b2adb8d5 --- /dev/null +++ b/apps/ios/Sources/Settings/VoiceWakeWordsSettingsView.swift @@ -0,0 +1,98 @@ +import SwiftUI +import Combine + +struct VoiceWakeWordsSettingsView: View { + @Environment(NodeAppModel.self) private var appModel + @State private var triggerWords: [String] = VoiceWakePreferences.loadTriggerWords() + @FocusState private var focusedTriggerIndex: Int? + @State private var syncTask: Task? + + var body: some View { + Form { + Section { + ForEach(self.triggerWords.indices, id: \.self) { index in + TextField("Wake word", text: self.binding(for: index)) + .textInputAutocapitalization(.never) + .autocorrectionDisabled() + .focused(self.$focusedTriggerIndex, equals: index) + .onSubmit { + self.commitTriggerWords() + } + } + .onDelete(perform: self.removeWords) + + Button { + self.addWord() + } label: { + Label("Add word", systemImage: "plus") + } + .disabled(self.triggerWords + .contains(where: { $0.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty })) + + Button("Reset defaults") { + self.triggerWords = VoiceWakePreferences.defaultTriggerWords + } + } header: { + Text("Wake Words") + } footer: { + Text( + "OpenClaw reacts when any trigger appears in a transcription. " + + "Keep them short to avoid false positives.") + } + } + .navigationTitle("Wake Words") + .toolbar { EditButton() } + .onAppear { + if self.triggerWords.isEmpty { + self.triggerWords = VoiceWakePreferences.defaultTriggerWords + self.commitTriggerWords() + } + } + .onChange(of: self.focusedTriggerIndex) { oldValue, newValue in + guard oldValue != nil, oldValue != newValue else { return } + self.commitTriggerWords() + } + .onReceive(NotificationCenter.default.publisher(for: UserDefaults.didChangeNotification)) { _ in + guard self.focusedTriggerIndex == nil else { return } + let updated = VoiceWakePreferences.loadTriggerWords() + if updated != self.triggerWords { + self.triggerWords = updated + } + } + } + + private func addWord() { + self.triggerWords.append("") + } + + private func removeWords(at offsets: IndexSet) { + self.triggerWords.remove(atOffsets: offsets) + if self.triggerWords.isEmpty { + self.triggerWords = VoiceWakePreferences.defaultTriggerWords + } + self.commitTriggerWords() + } + + private func binding(for index: Int) -> Binding { + Binding( + get: { + guard self.triggerWords.indices.contains(index) else { return "" } + return self.triggerWords[index] + }, + set: { newValue in + guard self.triggerWords.indices.contains(index) else { return } + self.triggerWords[index] = newValue + }) + } + + private func commitTriggerWords() { + VoiceWakePreferences.saveTriggerWords(self.triggerWords) + + let snapshot = VoiceWakePreferences.sanitizeTriggerWords(self.triggerWords) + self.syncTask?.cancel() + self.syncTask = Task { [snapshot, weak appModel = self.appModel] in + try? await Task.sleep(nanoseconds: 650_000_000) + await appModel?.setGlobalWakeWords(snapshot) + } + } +} diff --git a/apps/ios/Sources/Status/StatusPill.swift b/apps/ios/Sources/Status/StatusPill.swift new file mode 100644 index 0000000000000000000000000000000000000000..cd81c011bb1da4c279e595bec879d7ac81e1a97a --- /dev/null +++ b/apps/ios/Sources/Status/StatusPill.swift @@ -0,0 +1,127 @@ +import SwiftUI + +struct StatusPill: View { + @Environment(\.scenePhase) private var scenePhase + + enum GatewayState: Equatable { + case connected + case connecting + case error + case disconnected + + var title: String { + switch self { + case .connected: "Connected" + case .connecting: "Connecting…" + case .error: "Error" + case .disconnected: "Offline" + } + } + + var color: Color { + switch self { + case .connected: .green + case .connecting: .yellow + case .error: .red + case .disconnected: .gray + } + } + } + + struct Activity: Equatable { + var title: String + var systemImage: String + var tint: Color? + } + + var gateway: GatewayState + var voiceWakeEnabled: Bool + var activity: Activity? + var brighten: Bool = false + var onTap: () -> Void + + @State private var pulse: Bool = false + + var body: some View { + Button(action: self.onTap) { + HStack(spacing: 10) { + HStack(spacing: 8) { + Circle() + .fill(self.gateway.color) + .frame(width: 9, height: 9) + .scaleEffect(self.gateway == .connecting ? (self.pulse ? 1.15 : 0.85) : 1.0) + .opacity(self.gateway == .connecting ? (self.pulse ? 1.0 : 0.6) : 1.0) + + Text(self.gateway.title) + .font(.system(size: 13, weight: .semibold)) + .foregroundStyle(.primary) + } + + Divider() + .frame(height: 14) + .opacity(0.35) + + if let activity { + HStack(spacing: 6) { + Image(systemName: activity.systemImage) + .font(.system(size: 13, weight: .semibold)) + .foregroundStyle(activity.tint ?? .primary) + Text(activity.title) + .font(.system(size: 13, weight: .semibold)) + .foregroundStyle(.primary) + .lineLimit(1) + } + .transition(.opacity.combined(with: .move(edge: .top))) + } else { + Image(systemName: self.voiceWakeEnabled ? "mic.fill" : "mic.slash") + .font(.system(size: 13, weight: .semibold)) + .foregroundStyle(self.voiceWakeEnabled ? .primary : .secondary) + .accessibilityLabel(self.voiceWakeEnabled ? "Voice Wake enabled" : "Voice Wake disabled") + .transition(.opacity.combined(with: .move(edge: .top))) + } + } + .padding(.vertical, 8) + .padding(.horizontal, 12) + .background { + RoundedRectangle(cornerRadius: 14, style: .continuous) + .fill(.ultraThinMaterial) + .overlay { + RoundedRectangle(cornerRadius: 14, style: .continuous) + .strokeBorder(.white.opacity(self.brighten ? 0.24 : 0.18), lineWidth: 0.5) + } + .shadow(color: .black.opacity(0.25), radius: 12, y: 6) + } + } + .buttonStyle(.plain) + .accessibilityLabel("Status") + .accessibilityValue(self.accessibilityValue) + .onAppear { self.updatePulse(for: self.gateway, scenePhase: self.scenePhase) } + .onDisappear { self.pulse = false } + .onChange(of: self.gateway) { _, newValue in + self.updatePulse(for: newValue, scenePhase: self.scenePhase) + } + .onChange(of: self.scenePhase) { _, newValue in + self.updatePulse(for: self.gateway, scenePhase: newValue) + } + .animation(.easeInOut(duration: 0.18), value: self.activity?.title) + } + + private var accessibilityValue: String { + if let activity { + return "\(self.gateway.title), \(activity.title)" + } + return "\(self.gateway.title), Voice Wake \(self.voiceWakeEnabled ? "enabled" : "disabled")" + } + + private func updatePulse(for gateway: GatewayState, scenePhase: ScenePhase) { + guard gateway == .connecting, scenePhase == .active else { + withAnimation(.easeOut(duration: 0.2)) { self.pulse = false } + return + } + + guard !self.pulse else { return } + withAnimation(.easeInOut(duration: 0.9).repeatForever(autoreverses: true)) { + self.pulse = true + } + } +} diff --git a/apps/ios/Sources/Status/VoiceWakeToast.swift b/apps/ios/Sources/Status/VoiceWakeToast.swift new file mode 100644 index 0000000000000000000000000000000000000000..b7942f2036f5a55ed716578c5697a28fb0a84459 --- /dev/null +++ b/apps/ios/Sources/Status/VoiceWakeToast.swift @@ -0,0 +1,33 @@ +import SwiftUI + +struct VoiceWakeToast: View { + var command: String + var brighten: Bool = false + + var body: some View { + HStack(spacing: 10) { + Image(systemName: "mic.fill") + .font(.system(size: 14, weight: .semibold)) + .foregroundStyle(.primary) + + Text(self.command) + .font(.system(size: 14, weight: .semibold)) + .foregroundStyle(.primary) + .lineLimit(1) + .truncationMode(.tail) + } + .padding(.vertical, 10) + .padding(.horizontal, 12) + .background { + RoundedRectangle(cornerRadius: 14, style: .continuous) + .fill(.ultraThinMaterial) + .overlay { + RoundedRectangle(cornerRadius: 14, style: .continuous) + .strokeBorder(.white.opacity(self.brighten ? 0.24 : 0.18), lineWidth: 0.5) + } + .shadow(color: .black.opacity(0.25), radius: 12, y: 6) + } + .accessibilityLabel("Voice Wake") + .accessibilityValue(self.command) + } +} diff --git a/apps/ios/Sources/Voice/TalkModeManager.swift b/apps/ios/Sources/Voice/TalkModeManager.swift new file mode 100644 index 0000000000000000000000000000000000000000..d3adb49e1bc12f42c5b9fd7bb9ff70236756c703 --- /dev/null +++ b/apps/ios/Sources/Voice/TalkModeManager.swift @@ -0,0 +1,729 @@ +import AVFAudio +import OpenClawKit +import OpenClawProtocol +import Foundation +import Observation +import OSLog +import Speech + +@MainActor +@Observable +final class TalkModeManager: NSObject { + private typealias SpeechRequest = SFSpeechAudioBufferRecognitionRequest + private static let defaultModelIdFallback = "eleven_v3" + var isEnabled: Bool = false + var isListening: Bool = false + var isSpeaking: Bool = false + var statusText: String = "Off" + + private let audioEngine = AVAudioEngine() + private var speechRecognizer: SFSpeechRecognizer? + private var recognitionRequest: SFSpeechAudioBufferRecognitionRequest? + private var recognitionTask: SFSpeechRecognitionTask? + private var silenceTask: Task? + + private var lastHeard: Date? + private var lastTranscript: String = "" + private var lastSpokenText: String? + private var lastInterruptedAtSeconds: Double? + + private var defaultVoiceId: String? + private var currentVoiceId: String? + private var defaultModelId: String? + private var currentModelId: String? + private var voiceOverrideActive = false + private var modelOverrideActive = false + private var defaultOutputFormat: String? + private var apiKey: String? + private var voiceAliases: [String: String] = [:] + private var interruptOnSpeech: Bool = true + private var mainSessionKey: String = "main" + private var fallbackVoiceId: String? + private var lastPlaybackWasPCM: Bool = false + var pcmPlayer: PCMStreamingAudioPlaying = PCMStreamingAudioPlayer.shared + var mp3Player: StreamingAudioPlaying = StreamingAudioPlayer.shared + + private var gateway: GatewayNodeSession? + private let silenceWindow: TimeInterval = 0.7 + + private var chatSubscribedSessionKeys = Set() + + private let logger = Logger(subsystem: "bot.molt", category: "TalkMode") + + func attachGateway(_ gateway: GatewayNodeSession) { + self.gateway = gateway + } + + func updateMainSessionKey(_ sessionKey: String?) { + let trimmed = (sessionKey ?? "").trimmingCharacters(in: .whitespacesAndNewlines) + guard !trimmed.isEmpty else { return } + if SessionKey.isCanonicalMainSessionKey(self.mainSessionKey) { return } + self.mainSessionKey = trimmed + } + + func setEnabled(_ enabled: Bool) { + self.isEnabled = enabled + if enabled { + self.logger.info("enabled") + Task { await self.start() } + } else { + self.logger.info("disabled") + self.stop() + } + } + + func start() async { + guard self.isEnabled else { return } + if self.isListening { return } + + self.logger.info("start") + self.statusText = "Requesting permissions…" + let micOk = await Self.requestMicrophonePermission() + guard micOk else { + self.logger.warning("start blocked: microphone permission denied") + self.statusText = "Microphone permission denied" + return + } + let speechOk = await Self.requestSpeechPermission() + guard speechOk else { + self.logger.warning("start blocked: speech permission denied") + self.statusText = "Speech recognition permission denied" + return + } + + await self.reloadConfig() + do { + try Self.configureAudioSession() + try self.startRecognition() + self.isListening = true + self.statusText = "Listening" + self.startSilenceMonitor() + await self.subscribeChatIfNeeded(sessionKey: self.mainSessionKey) + self.logger.info("listening") + } catch { + self.isListening = false + self.statusText = "Start failed: \(error.localizedDescription)" + self.logger.error("start failed: \(error.localizedDescription, privacy: .public)") + } + } + + func stop() { + self.isEnabled = false + self.isListening = false + self.statusText = "Off" + self.lastTranscript = "" + self.lastHeard = nil + self.silenceTask?.cancel() + self.silenceTask = nil + self.stopRecognition() + self.stopSpeaking() + self.lastInterruptedAtSeconds = nil + TalkSystemSpeechSynthesizer.shared.stop() + do { + try AVAudioSession.sharedInstance().setActive(false, options: [.notifyOthersOnDeactivation]) + } catch { + self.logger.warning("audio session deactivate failed: \(error.localizedDescription, privacy: .public)") + } + Task { await self.unsubscribeAllChats() } + } + + func userTappedOrb() { + self.stopSpeaking() + } + + private func startRecognition() throws { + #if targetEnvironment(simulator) + throw NSError(domain: "TalkMode", code: 2, userInfo: [ + NSLocalizedDescriptionKey: "Talk mode is not supported on the iOS simulator", + ]) + #endif + + self.stopRecognition() + self.speechRecognizer = SFSpeechRecognizer() + guard let recognizer = self.speechRecognizer else { + throw NSError(domain: "TalkMode", code: 1, userInfo: [ + NSLocalizedDescriptionKey: "Speech recognizer unavailable", + ]) + } + + self.recognitionRequest = SFSpeechAudioBufferRecognitionRequest() + self.recognitionRequest?.shouldReportPartialResults = true + guard let request = self.recognitionRequest else { return } + + let input = self.audioEngine.inputNode + let format = input.outputFormat(forBus: 0) + guard format.sampleRate > 0, format.channelCount > 0 else { + throw NSError(domain: "TalkMode", code: 3, userInfo: [ + NSLocalizedDescriptionKey: "Invalid audio input format", + ]) + } + input.removeTap(onBus: 0) + let tapBlock = Self.makeAudioTapAppendCallback(request: request) + input.installTap(onBus: 0, bufferSize: 2048, format: format, block: tapBlock) + + self.audioEngine.prepare() + try self.audioEngine.start() + + self.recognitionTask = recognizer.recognitionTask(with: request) { [weak self] result, error in + guard let self else { return } + if let error { + if !self.isSpeaking { + self.statusText = "Speech error: \(error.localizedDescription)" + } + self.logger.debug("speech recognition error: \(error.localizedDescription, privacy: .public)") + } + guard let result else { return } + let transcript = result.bestTranscription.formattedString + Task { @MainActor in + await self.handleTranscript(transcript: transcript, isFinal: result.isFinal) + } + } + } + + private func stopRecognition() { + self.recognitionTask?.cancel() + self.recognitionTask = nil + self.recognitionRequest?.endAudio() + self.recognitionRequest = nil + self.audioEngine.inputNode.removeTap(onBus: 0) + self.audioEngine.stop() + self.speechRecognizer = nil + } + + private nonisolated static func makeAudioTapAppendCallback(request: SpeechRequest) -> AVAudioNodeTapBlock { + { buffer, _ in + request.append(buffer) + } + } + + private func handleTranscript(transcript: String, isFinal: Bool) async { + let trimmed = transcript.trimmingCharacters(in: .whitespacesAndNewlines) + if self.isSpeaking, self.interruptOnSpeech { + if self.shouldInterrupt(with: trimmed) { + self.stopSpeaking() + } + return + } + + guard self.isListening else { return } + if !trimmed.isEmpty { + self.lastTranscript = trimmed + self.lastHeard = Date() + } + if isFinal { + self.lastTranscript = trimmed + } + } + + private func startSilenceMonitor() { + self.silenceTask?.cancel() + self.silenceTask = Task { [weak self] in + guard let self else { return } + while self.isEnabled { + try? await Task.sleep(nanoseconds: 200_000_000) + await self.checkSilence() + } + } + } + + private func checkSilence() async { + guard self.isListening, !self.isSpeaking else { return } + let transcript = self.lastTranscript.trimmingCharacters(in: .whitespacesAndNewlines) + guard !transcript.isEmpty else { return } + guard let lastHeard else { return } + if Date().timeIntervalSince(lastHeard) < self.silenceWindow { return } + await self.finalizeTranscript(transcript) + } + + private func finalizeTranscript(_ transcript: String) async { + self.isListening = false + self.statusText = "Thinking…" + self.lastTranscript = "" + self.lastHeard = nil + self.stopRecognition() + + await self.reloadConfig() + let prompt = self.buildPrompt(transcript: transcript) + guard let gateway else { + self.statusText = "Gateway not connected" + self.logger.warning("finalize: gateway not connected") + await self.start() + return + } + + do { + let startedAt = Date().timeIntervalSince1970 + let sessionKey = self.mainSessionKey + await self.subscribeChatIfNeeded(sessionKey: sessionKey) + self.logger.info( + "chat.send start sessionKey=\(sessionKey, privacy: .public) chars=\(prompt.count, privacy: .public)") + let runId = try await self.sendChat(prompt, gateway: gateway) + self.logger.info("chat.send ok runId=\(runId, privacy: .public)") + let completion = await self.waitForChatCompletion(runId: runId, gateway: gateway, timeoutSeconds: 120) + if completion == .timeout { + self.logger.warning( + "chat completion timeout runId=\(runId, privacy: .public); attempting history fallback") + } else if completion == .aborted { + self.statusText = "Aborted" + self.logger.warning("chat completion aborted runId=\(runId, privacy: .public)") + await self.start() + return + } else if completion == .error { + self.statusText = "Chat error" + self.logger.warning("chat completion error runId=\(runId, privacy: .public)") + await self.start() + return + } + + guard let assistantText = try await self.waitForAssistantText( + gateway: gateway, + since: startedAt, + timeoutSeconds: completion == .final ? 12 : 25) + else { + self.statusText = "No reply" + self.logger.warning("assistant text timeout runId=\(runId, privacy: .public)") + await self.start() + return + } + self.logger.info("assistant text ok chars=\(assistantText.count, privacy: .public)") + await self.playAssistant(text: assistantText) + } catch { + self.statusText = "Talk failed: \(error.localizedDescription)" + self.logger.error("finalize failed: \(error.localizedDescription, privacy: .public)") + } + + await self.start() + } + + private func subscribeChatIfNeeded(sessionKey: String) async { + let key = sessionKey.trimmingCharacters(in: .whitespacesAndNewlines) + guard !key.isEmpty else { return } + guard let gateway else { return } + guard !self.chatSubscribedSessionKeys.contains(key) else { return } + + let payload = "{\"sessionKey\":\"\(key)\"}" + await gateway.sendEvent(event: "chat.subscribe", payloadJSON: payload) + self.chatSubscribedSessionKeys.insert(key) + self.logger.info("chat.subscribe ok sessionKey=\(key, privacy: .public)") + } + + private func unsubscribeAllChats() async { + guard let gateway else { return } + let keys = self.chatSubscribedSessionKeys + self.chatSubscribedSessionKeys.removeAll() + for key in keys { + let payload = "{\"sessionKey\":\"\(key)\"}" + await gateway.sendEvent(event: "chat.unsubscribe", payloadJSON: payload) + } + } + + private func buildPrompt(transcript: String) -> String { + let interrupted = self.lastInterruptedAtSeconds + self.lastInterruptedAtSeconds = nil + return TalkPromptBuilder.build(transcript: transcript, interruptedAtSeconds: interrupted) + } + + private enum ChatCompletionState: CustomStringConvertible { + case final + case aborted + case error + case timeout + + var description: String { + switch self { + case .final: "final" + case .aborted: "aborted" + case .error: "error" + case .timeout: "timeout" + } + } + } + + private func sendChat(_ message: String, gateway: GatewayNodeSession) async throws -> String { + struct SendResponse: Decodable { let runId: String } + let payload: [String: Any] = [ + "sessionKey": self.mainSessionKey, + "message": message, + "thinking": "low", + "timeoutMs": 30000, + "idempotencyKey": UUID().uuidString, + ] + let data = try JSONSerialization.data(withJSONObject: payload) + guard let json = String(bytes: data, encoding: .utf8) else { + throw NSError( + domain: "TalkModeManager", + code: 1, + userInfo: [NSLocalizedDescriptionKey: "Failed to encode chat payload"]) + } + let res = try await gateway.request(method: "chat.send", paramsJSON: json, timeoutSeconds: 30) + let decoded = try JSONDecoder().decode(SendResponse.self, from: res) + return decoded.runId + } + + private func waitForChatCompletion( + runId: String, + gateway: GatewayNodeSession, + timeoutSeconds: Int = 120) async -> ChatCompletionState + { + let stream = await gateway.subscribeServerEvents(bufferingNewest: 200) + return await withTaskGroup(of: ChatCompletionState.self) { group in + group.addTask { [runId] in + for await evt in stream { + if Task.isCancelled { return .timeout } + guard evt.event == "chat", let payload = evt.payload else { continue } + guard let chatEvent = try? GatewayPayloadDecoding.decode(payload, as: ChatEvent.self) else { + continue + } + guard chatEvent.runid == runId else { continue } + if let state = chatEvent.state.value as? String { + switch state { + case "final": return .final + case "aborted": return .aborted + case "error": return .error + default: break + } + } + } + return .timeout + } + group.addTask { + try? await Task.sleep(nanoseconds: UInt64(timeoutSeconds) * 1_000_000_000) + return .timeout + } + let result = await group.next() ?? .timeout + group.cancelAll() + return result + } + } + + private func waitForAssistantText( + gateway: GatewayNodeSession, + since: Double, + timeoutSeconds: Int) async throws -> String? + { + let deadline = Date().addingTimeInterval(TimeInterval(timeoutSeconds)) + while Date() < deadline { + if let text = try await self.fetchLatestAssistantText(gateway: gateway, since: since) { + return text + } + try? await Task.sleep(nanoseconds: 300_000_000) + } + return nil + } + + private func fetchLatestAssistantText(gateway: GatewayNodeSession, since: Double? = nil) async throws -> String? { + let res = try await gateway.request( + method: "chat.history", + paramsJSON: "{\"sessionKey\":\"\(self.mainSessionKey)\"}", + timeoutSeconds: 15) + guard let json = try JSONSerialization.jsonObject(with: res) as? [String: Any] else { return nil } + guard let messages = json["messages"] as? [[String: Any]] else { return nil } + for msg in messages.reversed() { + guard (msg["role"] as? String) == "assistant" else { continue } + if let since, let timestamp = msg["timestamp"] as? Double, + TalkHistoryTimestamp.isAfter(timestamp, sinceSeconds: since) == false + { + continue + } + guard let content = msg["content"] as? [[String: Any]] else { continue } + let text = content.compactMap { $0["text"] as? String }.joined(separator: "\n") + let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines) + if !trimmed.isEmpty { return trimmed } + } + return nil + } + + private func playAssistant(text: String) async { + let parsed = TalkDirectiveParser.parse(text) + let directive = parsed.directive + let cleaned = parsed.stripped.trimmingCharacters(in: .whitespacesAndNewlines) + guard !cleaned.isEmpty else { return } + + let requestedVoice = directive?.voiceId?.trimmingCharacters(in: .whitespacesAndNewlines) + let resolvedVoice = self.resolveVoiceAlias(requestedVoice) + if requestedVoice?.isEmpty == false, resolvedVoice == nil { + self.logger.warning("unknown voice alias \(requestedVoice ?? "?", privacy: .public)") + } + if let voice = resolvedVoice { + if directive?.once != true { + self.currentVoiceId = voice + self.voiceOverrideActive = true + } + } + if let model = directive?.modelId { + if directive?.once != true { + self.currentModelId = model + self.modelOverrideActive = true + } + } + + self.statusText = "Generating voice…" + self.isSpeaking = true + self.lastSpokenText = cleaned + + do { + let started = Date() + let language = ElevenLabsTTSClient.validatedLanguage(directive?.language) + + let resolvedKey = + (self.apiKey?.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty == false ? self.apiKey : nil) ?? + ProcessInfo.processInfo.environment["ELEVENLABS_API_KEY"] + let apiKey = resolvedKey?.trimmingCharacters(in: .whitespacesAndNewlines) + let preferredVoice = resolvedVoice ?? self.currentVoiceId ?? self.defaultVoiceId + let voiceId: String? = if let apiKey, !apiKey.isEmpty { + await self.resolveVoiceId(preferred: preferredVoice, apiKey: apiKey) + } else { + nil + } + let canUseElevenLabs = (voiceId?.isEmpty == false) && (apiKey?.isEmpty == false) + + if canUseElevenLabs, let voiceId, let apiKey { + let desiredOutputFormat = (directive?.outputFormat ?? self.defaultOutputFormat)? + .trimmingCharacters(in: .whitespacesAndNewlines) + let requestedOutputFormat = (desiredOutputFormat?.isEmpty == false) ? desiredOutputFormat : nil + let outputFormat = ElevenLabsTTSClient.validatedOutputFormat(requestedOutputFormat ?? "pcm_44100") + if outputFormat == nil, let requestedOutputFormat { + self.logger.warning( + "talk output_format unsupported for local playback: \(requestedOutputFormat, privacy: .public)") + } + + let modelId = directive?.modelId ?? self.currentModelId ?? self.defaultModelId + func makeRequest(outputFormat: String?) -> ElevenLabsTTSRequest { + ElevenLabsTTSRequest( + text: cleaned, + modelId: modelId, + outputFormat: outputFormat, + speed: TalkTTSValidation.resolveSpeed(speed: directive?.speed, rateWPM: directive?.rateWPM), + stability: TalkTTSValidation.validatedStability(directive?.stability, modelId: modelId), + similarity: TalkTTSValidation.validatedUnit(directive?.similarity), + style: TalkTTSValidation.validatedUnit(directive?.style), + speakerBoost: directive?.speakerBoost, + seed: TalkTTSValidation.validatedSeed(directive?.seed), + normalize: ElevenLabsTTSClient.validatedNormalize(directive?.normalize), + language: language, + latencyTier: TalkTTSValidation.validatedLatencyTier(directive?.latencyTier)) + } + + let request = makeRequest(outputFormat: outputFormat) + + let client = ElevenLabsTTSClient(apiKey: apiKey) + let stream = client.streamSynthesize(voiceId: voiceId, request: request) + + if self.interruptOnSpeech { + do { + try self.startRecognition() + } catch { + self.logger.warning( + "startRecognition during speak failed: \(error.localizedDescription, privacy: .public)") + } + } + + self.statusText = "Speaking…" + let sampleRate = TalkTTSValidation.pcmSampleRate(from: outputFormat) + let result: StreamingPlaybackResult + if let sampleRate { + self.lastPlaybackWasPCM = true + var playback = await self.pcmPlayer.play(stream: stream, sampleRate: sampleRate) + if !playback.finished, playback.interruptedAt == nil { + let mp3Format = ElevenLabsTTSClient.validatedOutputFormat("mp3_44100") + self.logger.warning("pcm playback failed; retrying mp3") + self.lastPlaybackWasPCM = false + let mp3Stream = client.streamSynthesize( + voiceId: voiceId, + request: makeRequest(outputFormat: mp3Format)) + playback = await self.mp3Player.play(stream: mp3Stream) + } + result = playback + } else { + self.lastPlaybackWasPCM = false + result = await self.mp3Player.play(stream: stream) + } + let duration = Date().timeIntervalSince(started) + self.logger.info("elevenlabs stream finished=\(result.finished, privacy: .public) dur=\(duration, privacy: .public)s") + if !result.finished, let interruptedAt = result.interruptedAt { + self.lastInterruptedAtSeconds = interruptedAt + } + } else { + self.logger.warning("tts unavailable; falling back to system voice (missing key or voiceId)") + if self.interruptOnSpeech { + do { + try self.startRecognition() + } catch { + self.logger.warning( + "startRecognition during speak failed: \(error.localizedDescription, privacy: .public)") + } + } + self.statusText = "Speaking (System)…" + try await TalkSystemSpeechSynthesizer.shared.speak(text: cleaned, language: language) + } + } catch { + self.logger.error( + "tts failed: \(error.localizedDescription, privacy: .public); falling back to system voice") + do { + if self.interruptOnSpeech { + do { + try self.startRecognition() + } catch { + self.logger.warning( + "startRecognition during speak failed: \(error.localizedDescription, privacy: .public)") + } + } + self.statusText = "Speaking (System)…" + let language = ElevenLabsTTSClient.validatedLanguage(directive?.language) + try await TalkSystemSpeechSynthesizer.shared.speak(text: cleaned, language: language) + } catch { + self.statusText = "Speak failed: \(error.localizedDescription)" + self.logger.error("system voice failed: \(error.localizedDescription, privacy: .public)") + } + } + + self.stopRecognition() + self.isSpeaking = false + } + + private func stopSpeaking(storeInterruption: Bool = true) { + guard self.isSpeaking else { return } + let interruptedAt = self.lastPlaybackWasPCM + ? self.pcmPlayer.stop() + : self.mp3Player.stop() + if storeInterruption { + self.lastInterruptedAtSeconds = interruptedAt + } + _ = self.lastPlaybackWasPCM + ? self.mp3Player.stop() + : self.pcmPlayer.stop() + TalkSystemSpeechSynthesizer.shared.stop() + self.isSpeaking = false + } + + private func shouldInterrupt(with transcript: String) -> Bool { + let trimmed = transcript.trimmingCharacters(in: .whitespacesAndNewlines) + guard trimmed.count >= 3 else { return false } + if let spoken = self.lastSpokenText?.lowercased(), spoken.contains(trimmed.lowercased()) { + return false + } + return true + } + + private func resolveVoiceAlias(_ value: String?) -> String? { + let trimmed = (value ?? "").trimmingCharacters(in: .whitespacesAndNewlines) + guard !trimmed.isEmpty else { return nil } + let normalized = trimmed.lowercased() + if let mapped = self.voiceAliases[normalized] { return mapped } + if self.voiceAliases.values.contains(where: { $0.caseInsensitiveCompare(trimmed) == .orderedSame }) { + return trimmed + } + return Self.isLikelyVoiceId(trimmed) ? trimmed : nil + } + + private func resolveVoiceId(preferred: String?, apiKey: String) async -> String? { + let trimmed = preferred?.trimmingCharacters(in: .whitespacesAndNewlines) ?? "" + if !trimmed.isEmpty { + if let resolved = self.resolveVoiceAlias(trimmed) { return resolved } + self.logger.warning("unknown voice alias \(trimmed, privacy: .public)") + } + if let fallbackVoiceId { return fallbackVoiceId } + + do { + let voices = try await ElevenLabsTTSClient(apiKey: apiKey).listVoices() + guard let first = voices.first else { + self.logger.warning("elevenlabs voices list empty") + return nil + } + self.fallbackVoiceId = first.voiceId + if self.defaultVoiceId == nil { + self.defaultVoiceId = first.voiceId + } + if !self.voiceOverrideActive { + self.currentVoiceId = first.voiceId + } + let name = first.name ?? "unknown" + self.logger + .info("default voice selected \(name, privacy: .public) (\(first.voiceId, privacy: .public))") + return first.voiceId + } catch { + self.logger.error("elevenlabs list voices failed: \(error.localizedDescription, privacy: .public)") + return nil + } + } + + private static func isLikelyVoiceId(_ value: String) -> Bool { + guard value.count >= 10 else { return false } + return value.allSatisfy { $0.isLetter || $0.isNumber || $0 == "-" || $0 == "_" } + } + + private func reloadConfig() async { + guard let gateway else { return } + do { + let res = try await gateway.request(method: "config.get", paramsJSON: "{}", timeoutSeconds: 8) + guard let json = try JSONSerialization.jsonObject(with: res) as? [String: Any] else { return } + guard let config = json["config"] as? [String: Any] else { return } + let talk = config["talk"] as? [String: Any] + let session = config["session"] as? [String: Any] + let mainKey = SessionKey.normalizeMainKey(session?["mainKey"] as? String) + if !SessionKey.isCanonicalMainSessionKey(self.mainSessionKey) { + self.mainSessionKey = mainKey + } + self.defaultVoiceId = (talk?["voiceId"] as? String)?.trimmingCharacters(in: .whitespacesAndNewlines) + if let aliases = talk?["voiceAliases"] as? [String: Any] { + var resolved: [String: String] = [:] + for (key, value) in aliases { + guard let id = value as? String else { continue } + let normalizedKey = key.trimmingCharacters(in: .whitespacesAndNewlines).lowercased() + let trimmedId = id.trimmingCharacters(in: .whitespacesAndNewlines) + guard !normalizedKey.isEmpty, !trimmedId.isEmpty else { continue } + resolved[normalizedKey] = trimmedId + } + self.voiceAliases = resolved + } else { + self.voiceAliases = [:] + } + if !self.voiceOverrideActive { + self.currentVoiceId = self.defaultVoiceId + } + let model = (talk?["modelId"] as? String)?.trimmingCharacters(in: .whitespacesAndNewlines) + self.defaultModelId = (model?.isEmpty == false) ? model : Self.defaultModelIdFallback + if !self.modelOverrideActive { + self.currentModelId = self.defaultModelId + } + self.defaultOutputFormat = (talk?["outputFormat"] as? String)? + .trimmingCharacters(in: .whitespacesAndNewlines) + self.apiKey = (talk?["apiKey"] as? String)?.trimmingCharacters(in: .whitespacesAndNewlines) + if let interrupt = talk?["interruptOnSpeech"] as? Bool { + self.interruptOnSpeech = interrupt + } + } catch { + self.defaultModelId = Self.defaultModelIdFallback + if !self.modelOverrideActive { + self.currentModelId = self.defaultModelId + } + } + } + + private static func configureAudioSession() throws { + let session = AVAudioSession.sharedInstance() + try session.setCategory(.playAndRecord, mode: .voiceChat, options: [ + .duckOthers, + .mixWithOthers, + .allowBluetoothHFP, + .defaultToSpeaker, + ]) + try session.setActive(true, options: []) + } + + private nonisolated static func requestMicrophonePermission() async -> Bool { + await withCheckedContinuation(isolation: nil) { cont in + AVAudioApplication.requestRecordPermission { ok in + cont.resume(returning: ok) + } + } + } + + private nonisolated static func requestSpeechPermission() async -> Bool { + await withCheckedContinuation(isolation: nil) { cont in + SFSpeechRecognizer.requestAuthorization { status in + cont.resume(returning: status == .authorized) + } + } + } +} diff --git a/apps/ios/Sources/Voice/TalkOrbOverlay.swift b/apps/ios/Sources/Voice/TalkOrbOverlay.swift new file mode 100644 index 0000000000000000000000000000000000000000..cce8c1c6110a1e0a344962d442403527a62d1800 --- /dev/null +++ b/apps/ios/Sources/Voice/TalkOrbOverlay.swift @@ -0,0 +1,70 @@ +import SwiftUI + +struct TalkOrbOverlay: View { + @Environment(NodeAppModel.self) private var appModel + @State private var pulse: Bool = false + + var body: some View { + let seam = self.appModel.seamColor + let status = self.appModel.talkMode.statusText.trimmingCharacters(in: .whitespacesAndNewlines) + + VStack(spacing: 14) { + ZStack { + Circle() + .stroke(seam.opacity(0.26), lineWidth: 2) + .frame(width: 320, height: 320) + .scaleEffect(self.pulse ? 1.15 : 0.96) + .opacity(self.pulse ? 0.0 : 1.0) + .animation(.easeOut(duration: 1.3).repeatForever(autoreverses: false), value: self.pulse) + + Circle() + .stroke(seam.opacity(0.18), lineWidth: 2) + .frame(width: 320, height: 320) + .scaleEffect(self.pulse ? 1.45 : 1.02) + .opacity(self.pulse ? 0.0 : 0.9) + .animation(.easeOut(duration: 1.9).repeatForever(autoreverses: false).delay(0.2), value: self.pulse) + + Circle() + .fill( + RadialGradient( + colors: [ + seam.opacity(0.95), + seam.opacity(0.40), + Color.black.opacity(0.55), + ], + center: .center, + startRadius: 1, + endRadius: 112)) + .frame(width: 190, height: 190) + .overlay( + Circle() + .stroke(seam.opacity(0.35), lineWidth: 1)) + .shadow(color: seam.opacity(0.32), radius: 26, x: 0, y: 0) + .shadow(color: Color.black.opacity(0.50), radius: 22, x: 0, y: 10) + } + .contentShape(Circle()) + .onTapGesture { + self.appModel.talkMode.userTappedOrb() + } + + if !status.isEmpty, status != "Off" { + Text(status) + .font(.system(.footnote, design: .rounded).weight(.semibold)) + .foregroundStyle(Color.white.opacity(0.92)) + .padding(.horizontal, 12) + .padding(.vertical, 8) + .background( + Capsule() + .fill(Color.black.opacity(0.40)) + .overlay( + Capsule().stroke(seam.opacity(0.22), lineWidth: 1))) + } + } + .padding(28) + .onAppear { + self.pulse = true + } + .accessibilityElement(children: .combine) + .accessibilityLabel("Talk Mode \(status)") + } +} diff --git a/apps/ios/Sources/Voice/VoiceTab.swift b/apps/ios/Sources/Voice/VoiceTab.swift new file mode 100644 index 0000000000000000000000000000000000000000..4fedd0ce9aa3218943548b2978d9332fd2640f6d --- /dev/null +++ b/apps/ios/Sources/Voice/VoiceTab.swift @@ -0,0 +1,46 @@ +import SwiftUI + +struct VoiceTab: View { + @Environment(NodeAppModel.self) private var appModel + @Environment(VoiceWakeManager.self) private var voiceWake + @AppStorage("voiceWake.enabled") private var voiceWakeEnabled: Bool = false + @AppStorage("talk.enabled") private var talkEnabled: Bool = false + + var body: some View { + NavigationStack { + List { + Section("Status") { + LabeledContent("Voice Wake", value: self.voiceWakeEnabled ? "Enabled" : "Disabled") + LabeledContent("Listener", value: self.voiceWake.isListening ? "Listening" : "Idle") + Text(self.voiceWake.statusText) + .font(.footnote) + .foregroundStyle(.secondary) + LabeledContent("Talk Mode", value: self.talkEnabled ? "Enabled" : "Disabled") + } + + Section("Notes") { + let triggers = self.voiceWake.activeTriggerWords + Group { + if triggers.isEmpty { + Text("Add wake words in Settings.") + } else if triggers.count == 1 { + Text("Say “\(triggers[0]) …” to trigger.") + } else if triggers.count == 2 { + Text("Say “\(triggers[0]) …” or “\(triggers[1]) …” to trigger.") + } else { + Text("Say “\(triggers.joined(separator: " …”, “")) …” to trigger.") + } + } + .foregroundStyle(.secondary) + } + } + .navigationTitle("Voice") + .onChange(of: self.voiceWakeEnabled) { _, newValue in + self.appModel.setVoiceWakeEnabled(newValue) + } + .onChange(of: self.talkEnabled) { _, newValue in + self.appModel.setTalkEnabled(newValue) + } + } + } +} diff --git a/apps/ios/Sources/Voice/VoiceWakeManager.swift b/apps/ios/Sources/Voice/VoiceWakeManager.swift new file mode 100644 index 0000000000000000000000000000000000000000..771b5a77a66fee4d0157a59e802fa1f4175adb2c --- /dev/null +++ b/apps/ios/Sources/Voice/VoiceWakeManager.swift @@ -0,0 +1,389 @@ +import AVFAudio +import Foundation +import Observation +import Speech +import SwabbleKit + +private func makeAudioTapEnqueueCallback(queue: AudioBufferQueue) -> @Sendable (AVAudioPCMBuffer, AVAudioTime) -> Void { + { buffer, _ in + // This callback is invoked on a realtime audio thread/queue. Keep it tiny and nonisolated. + queue.enqueueCopy(of: buffer) + } +} + +private final class AudioBufferQueue: @unchecked Sendable { + private let lock = NSLock() + private var buffers: [AVAudioPCMBuffer] = [] + + func enqueueCopy(of buffer: AVAudioPCMBuffer) { + guard let copy = buffer.deepCopy() else { return } + self.lock.lock() + self.buffers.append(copy) + self.lock.unlock() + } + + func drain() -> [AVAudioPCMBuffer] { + self.lock.lock() + let drained = self.buffers + self.buffers.removeAll(keepingCapacity: true) + self.lock.unlock() + return drained + } + + func clear() { + self.lock.lock() + self.buffers.removeAll(keepingCapacity: false) + self.lock.unlock() + } +} + +extension AVAudioPCMBuffer { + fileprivate func deepCopy() -> AVAudioPCMBuffer? { + let format = self.format + let frameLength = self.frameLength + guard let copy = AVAudioPCMBuffer(pcmFormat: format, frameCapacity: frameLength) else { + return nil + } + copy.frameLength = frameLength + + if let src = self.floatChannelData, let dst = copy.floatChannelData { + let channels = Int(format.channelCount) + let frames = Int(frameLength) + for ch in 0..? + + private var lastDispatched: String? + private var onCommand: (@Sendable (String) async -> Void)? + private var userDefaultsObserver: NSObjectProtocol? + + override init() { + super.init() + self.triggerWords = VoiceWakePreferences.loadTriggerWords() + self.userDefaultsObserver = NotificationCenter.default.addObserver( + forName: UserDefaults.didChangeNotification, + object: UserDefaults.standard, + queue: .main, + using: { [weak self] _ in + Task { @MainActor in + self?.handleUserDefaultsDidChange() + } + }) + } + + @MainActor deinit { + if let userDefaultsObserver = self.userDefaultsObserver { + NotificationCenter.default.removeObserver(userDefaultsObserver) + } + } + + var activeTriggerWords: [String] { + VoiceWakePreferences.sanitizeTriggerWords(self.triggerWords) + } + + private func handleUserDefaultsDidChange() { + let updated = VoiceWakePreferences.loadTriggerWords() + if updated != self.triggerWords { + self.triggerWords = updated + } + } + + func configure(onCommand: @escaping @Sendable (String) async -> Void) { + self.onCommand = onCommand + } + + func setEnabled(_ enabled: Bool) { + self.isEnabled = enabled + if enabled { + Task { await self.start() } + } else { + self.stop() + } + } + + func start() async { + guard self.isEnabled else { return } + if self.isListening { return } + + if ProcessInfo.processInfo.environment["SIMULATOR_DEVICE_NAME"] != nil || + ProcessInfo.processInfo.environment["SIMULATOR_UDID"] != nil + { + // The iOS Simulator’s audio stack is unreliable for long-running microphone capture. + // (We’ve observed CoreAudio deadlocks after TCC permission prompts.) + self.isListening = false + self.statusText = "Voice Wake isn’t supported on Simulator" + return + } + + self.statusText = "Requesting permissions…" + + let micOk = await Self.requestMicrophonePermission() + guard micOk else { + self.statusText = "Microphone permission denied" + self.isListening = false + return + } + + let speechOk = await Self.requestSpeechPermission() + guard speechOk else { + self.statusText = "Speech recognition permission denied" + self.isListening = false + return + } + + self.speechRecognizer = SFSpeechRecognizer() + guard self.speechRecognizer != nil else { + self.statusText = "Speech recognizer unavailable" + self.isListening = false + return + } + + do { + try Self.configureAudioSession() + try self.startRecognition() + self.isListening = true + self.statusText = "Listening" + } catch { + self.isListening = false + self.statusText = "Start failed: \(error.localizedDescription)" + } + } + + func stop() { + self.isEnabled = false + self.isListening = false + self.statusText = "Off" + + self.tapDrainTask?.cancel() + self.tapDrainTask = nil + self.tapQueue?.clear() + self.tapQueue = nil + + self.recognitionTask?.cancel() + self.recognitionTask = nil + self.recognitionRequest = nil + + if self.audioEngine.isRunning { + self.audioEngine.stop() + self.audioEngine.inputNode.removeTap(onBus: 0) + } + + try? AVAudioSession.sharedInstance().setActive(false, options: .notifyOthersOnDeactivation) + } + + /// Temporarily releases the microphone so other subsystems (e.g. camera video capture) can record audio. + /// Returns `true` when listening was active and was suspended. + func suspendForExternalAudioCapture() -> Bool { + guard self.isEnabled, self.isListening else { return false } + + self.isListening = false + self.statusText = "Paused" + + self.tapDrainTask?.cancel() + self.tapDrainTask = nil + self.tapQueue?.clear() + self.tapQueue = nil + + self.recognitionTask?.cancel() + self.recognitionTask = nil + self.recognitionRequest = nil + + if self.audioEngine.isRunning { + self.audioEngine.stop() + self.audioEngine.inputNode.removeTap(onBus: 0) + } + + try? AVAudioSession.sharedInstance().setActive(false, options: .notifyOthersOnDeactivation) + return true + } + + func resumeAfterExternalAudioCapture(wasSuspended: Bool) { + guard wasSuspended else { return } + Task { await self.start() } + } + + private func startRecognition() throws { + self.recognitionTask?.cancel() + self.recognitionTask = nil + self.tapDrainTask?.cancel() + self.tapDrainTask = nil + self.tapQueue?.clear() + self.tapQueue = nil + + let request = SFSpeechAudioBufferRecognitionRequest() + request.shouldReportPartialResults = true + self.recognitionRequest = request + + let inputNode = self.audioEngine.inputNode + inputNode.removeTap(onBus: 0) + + let recordingFormat = inputNode.outputFormat(forBus: 0) + + let queue = AudioBufferQueue() + self.tapQueue = queue + let tapBlock: @Sendable (AVAudioPCMBuffer, AVAudioTime) -> Void = makeAudioTapEnqueueCallback(queue: queue) + inputNode.installTap( + onBus: 0, + bufferSize: 1024, + format: recordingFormat, + block: tapBlock) + + self.audioEngine.prepare() + try self.audioEngine.start() + + let handler = self.makeRecognitionResultHandler() + self.recognitionTask = self.speechRecognizer?.recognitionTask(with: request, resultHandler: handler) + + self.tapDrainTask = Task { [weak self] in + guard let self, let queue = self.tapQueue else { return } + while !Task.isCancelled { + try? await Task.sleep(nanoseconds: 40_000_000) + let drained = queue.drain() + if drained.isEmpty { continue } + for buf in drained { + request.append(buf) + } + } + } + } + + private nonisolated func makeRecognitionResultHandler() -> @Sendable (SFSpeechRecognitionResult?, Error?) -> Void { + { [weak self] result, error in + let transcript = result?.bestTranscription.formattedString + let segments = result.flatMap { result in + transcript.map { WakeWordSpeechSegments.from(transcription: result.bestTranscription, transcript: $0) } + } ?? [] + let errorText = error?.localizedDescription + + Task { @MainActor in + self?.handleRecognitionCallback(transcript: transcript, segments: segments, errorText: errorText) + } + } + } + + private func handleRecognitionCallback(transcript: String?, segments: [WakeWordSegment], errorText: String?) { + if let errorText { + self.statusText = "Recognizer error: \(errorText)" + self.isListening = false + + let shouldRestart = self.isEnabled + if shouldRestart { + Task { + try? await Task.sleep(nanoseconds: 700_000_000) + await self.start() + } + } + return + } + + guard let transcript else { return } + guard let cmd = self.extractCommand(from: transcript, segments: segments) else { return } + + if cmd == self.lastDispatched { return } + self.lastDispatched = cmd + self.lastTriggeredCommand = cmd + self.statusText = "Triggered" + + Task { [weak self] in + guard let self else { return } + await self.onCommand?(cmd) + await self.startIfEnabled() + } + } + + private func startIfEnabled() async { + let shouldRestart = self.isEnabled + if shouldRestart { + await self.start() + } + } + + private func extractCommand(from transcript: String, segments: [WakeWordSegment]) -> String? { + Self.extractCommand(from: transcript, segments: segments, triggers: self.activeTriggerWords) + } + + nonisolated static func extractCommand( + from transcript: String, + segments: [WakeWordSegment], + triggers: [String], + minPostTriggerGap: TimeInterval = 0.45) -> String? + { + let config = WakeWordGateConfig(triggers: triggers, minPostTriggerGap: minPostTriggerGap) + return WakeWordGate.match(transcript: transcript, segments: segments, config: config)?.command + } + + private static func configureAudioSession() throws { + let session = AVAudioSession.sharedInstance() + try session.setCategory(.playAndRecord, mode: .measurement, options: [ + .duckOthers, + .mixWithOthers, + .allowBluetoothHFP, + .defaultToSpeaker, + ]) + try session.setActive(true, options: []) + } + + private nonisolated static func requestMicrophonePermission() async -> Bool { + await withCheckedContinuation(isolation: nil) { cont in + AVAudioApplication.requestRecordPermission { ok in + cont.resume(returning: ok) + } + } + } + + private nonisolated static func requestSpeechPermission() async -> Bool { + await withCheckedContinuation(isolation: nil) { cont in + SFSpeechRecognizer.requestAuthorization { status in + cont.resume(returning: status == .authorized) + } + } + } +} + +#if DEBUG +extension VoiceWakeManager { + func _test_handleRecognitionCallback(transcript: String?, segments: [WakeWordSegment], errorText: String?) { + self.handleRecognitionCallback(transcript: transcript, segments: segments, errorText: errorText) + } +} +#endif diff --git a/apps/ios/Sources/Voice/VoiceWakePreferences.swift b/apps/ios/Sources/Voice/VoiceWakePreferences.swift new file mode 100644 index 0000000000000000000000000000000000000000..56762b515e2c70f79b4ddb148f2078c75a5308cc --- /dev/null +++ b/apps/ios/Sources/Voice/VoiceWakePreferences.swift @@ -0,0 +1,44 @@ +import Foundation + +enum VoiceWakePreferences { + static let enabledKey = "voiceWake.enabled" + static let triggerWordsKey = "voiceWake.triggerWords" + + // Keep defaults aligned with the mac app. + static let defaultTriggerWords: [String] = ["openclaw", "claude"] + static let maxWords = 32 + static let maxWordLength = 64 + + static func decodeGatewayTriggers(from payloadJSON: String) -> [String]? { + guard let data = payloadJSON.data(using: .utf8) else { return nil } + return self.decodeGatewayTriggers(from: data) + } + + static func decodeGatewayTriggers(from data: Data) -> [String]? { + struct Payload: Decodable { var triggers: [String] } + guard let decoded = try? JSONDecoder().decode(Payload.self, from: data) else { return nil } + return self.sanitizeTriggerWords(decoded.triggers) + } + + static func loadTriggerWords(defaults: UserDefaults = .standard) -> [String] { + defaults.stringArray(forKey: self.triggerWordsKey) ?? self.defaultTriggerWords + } + + static func saveTriggerWords(_ words: [String], defaults: UserDefaults = .standard) { + defaults.set(words, forKey: self.triggerWordsKey) + } + + static func sanitizeTriggerWords(_ words: [String]) -> [String] { + let cleaned = words + .map { $0.trimmingCharacters(in: .whitespacesAndNewlines) } + .filter { !$0.isEmpty } + .prefix(Self.maxWords) + .map { String($0.prefix(Self.maxWordLength)) } + return cleaned.isEmpty ? Self.defaultTriggerWords : cleaned + } + + static func displayString(for words: [String]) -> String { + let sanitized = self.sanitizeTriggerWords(words) + return sanitized.joined(separator: ", ") + } +} diff --git a/apps/ios/SwiftSources.input.xcfilelist b/apps/ios/SwiftSources.input.xcfilelist new file mode 100644 index 0000000000000000000000000000000000000000..4952019c773ab37ef158a5882375d64edf1d5532 --- /dev/null +++ b/apps/ios/SwiftSources.input.xcfilelist @@ -0,0 +1,60 @@ +Sources/Gateway/GatewayConnectionController.swift +Sources/Gateway/GatewayDiscoveryDebugLogView.swift +Sources/Gateway/GatewayDiscoveryModel.swift +Sources/Gateway/GatewaySettingsStore.swift +Sources/Gateway/KeychainStore.swift +Sources/Camera/CameraController.swift +Sources/Chat/ChatSheet.swift +Sources/Chat/IOSGatewayChatTransport.swift +Sources/OpenClawApp.swift +Sources/Location/LocationService.swift +Sources/Model/NodeAppModel.swift +Sources/RootCanvas.swift +Sources/RootTabs.swift +Sources/Screen/ScreenController.swift +Sources/Screen/ScreenRecordService.swift +Sources/Screen/ScreenTab.swift +Sources/Screen/ScreenWebView.swift +Sources/SessionKey.swift +Sources/Settings/SettingsNetworkingHelpers.swift +Sources/Settings/SettingsTab.swift +Sources/Settings/VoiceWakeWordsSettingsView.swift +Sources/Status/StatusPill.swift +Sources/Status/VoiceWakeToast.swift +Sources/Voice/VoiceTab.swift +Sources/Voice/VoiceWakeManager.swift +Sources/Voice/VoiceWakePreferences.swift +../shared/OpenClawKit/Sources/OpenClawChatUI/ChatComposer.swift +../shared/OpenClawKit/Sources/OpenClawChatUI/ChatMarkdownRenderer.swift +../shared/OpenClawKit/Sources/OpenClawChatUI/ChatMarkdownPreprocessor.swift +../shared/OpenClawKit/Sources/OpenClawChatUI/ChatMessageViews.swift +../shared/OpenClawKit/Sources/OpenClawChatUI/ChatModels.swift +../shared/OpenClawKit/Sources/OpenClawChatUI/ChatPayloadDecoding.swift +../shared/OpenClawKit/Sources/OpenClawChatUI/ChatSessions.swift +../shared/OpenClawKit/Sources/OpenClawChatUI/ChatSheets.swift +../shared/OpenClawKit/Sources/OpenClawChatUI/ChatTheme.swift +../shared/OpenClawKit/Sources/OpenClawChatUI/ChatTransport.swift +../shared/OpenClawKit/Sources/OpenClawChatUI/ChatView.swift +../shared/OpenClawKit/Sources/OpenClawChatUI/ChatViewModel.swift +../shared/OpenClawKit/Sources/OpenClawKit/AnyCodable.swift +../shared/OpenClawKit/Sources/OpenClawKit/BonjourEscapes.swift +../shared/OpenClawKit/Sources/OpenClawKit/BonjourTypes.swift +../shared/OpenClawKit/Sources/OpenClawKit/BridgeFrames.swift +../shared/OpenClawKit/Sources/OpenClawKit/CameraCommands.swift +../shared/OpenClawKit/Sources/OpenClawKit/CanvasA2UIAction.swift +../shared/OpenClawKit/Sources/OpenClawKit/CanvasA2UICommands.swift +../shared/OpenClawKit/Sources/OpenClawKit/CanvasA2UIJSONL.swift +../shared/OpenClawKit/Sources/OpenClawKit/CanvasCommandParams.swift +../shared/OpenClawKit/Sources/OpenClawKit/CanvasCommands.swift +../shared/OpenClawKit/Sources/OpenClawKit/Capabilities.swift +../shared/OpenClawKit/Sources/OpenClawKit/OpenClawKitResources.swift +../shared/OpenClawKit/Sources/OpenClawKit/DeepLinks.swift +../shared/OpenClawKit/Sources/OpenClawKit/JPEGTranscoder.swift +../shared/OpenClawKit/Sources/OpenClawKit/NodeError.swift +../shared/OpenClawKit/Sources/OpenClawKit/ScreenCommands.swift +../shared/OpenClawKit/Sources/OpenClawKit/StoragePaths.swift +../shared/OpenClawKit/Sources/OpenClawKit/SystemCommands.swift +../shared/OpenClawKit/Sources/OpenClawKit/TalkDirective.swift +../../Swabble/Sources/SwabbleKit/WakeWordGate.swift +Sources/Voice/TalkModeManager.swift +Sources/Voice/TalkOrbOverlay.swift diff --git a/apps/ios/Tests/AppCoverageTests.swift b/apps/ios/Tests/AppCoverageTests.swift new file mode 100644 index 0000000000000000000000000000000000000000..33c71cccd05ed095b35e428605e2e0ac75d147c5 --- /dev/null +++ b/apps/ios/Tests/AppCoverageTests.swift @@ -0,0 +1,31 @@ +import SwiftUI +import Testing +@testable import OpenClaw + +@Suite struct AppCoverageTests { + @Test @MainActor func nodeAppModelUpdatesBackgroundedState() { + let appModel = NodeAppModel() + + appModel.setScenePhase(.background) + #expect(appModel.isBackgrounded == true) + + appModel.setScenePhase(.inactive) + #expect(appModel.isBackgrounded == false) + + appModel.setScenePhase(.active) + #expect(appModel.isBackgrounded == false) + } + + @Test @MainActor func voiceWakeStartReportsUnsupportedOnSimulator() async { + let voiceWake = VoiceWakeManager() + voiceWake.isEnabled = true + + await voiceWake.start() + + #expect(voiceWake.isListening == false) + #expect(voiceWake.statusText.contains("Simulator")) + + voiceWake.stop() + #expect(voiceWake.statusText == "Off") + } +} diff --git a/apps/ios/Tests/CameraControllerClampTests.swift b/apps/ios/Tests/CameraControllerClampTests.swift new file mode 100644 index 0000000000000000000000000000000000000000..791010d11b05aa02cb16ab99aab837c4c12beacd --- /dev/null +++ b/apps/ios/Tests/CameraControllerClampTests.swift @@ -0,0 +1,24 @@ +import Testing +@testable import OpenClaw + +@Suite struct CameraControllerClampTests { + @Test func clampQualityDefaultsAndBounds() { + #expect(CameraController.clampQuality(nil) == 0.9) + #expect(CameraController.clampQuality(0.0) == 0.05) + #expect(CameraController.clampQuality(0.049) == 0.05) + #expect(CameraController.clampQuality(0.05) == 0.05) + #expect(CameraController.clampQuality(0.5) == 0.5) + #expect(CameraController.clampQuality(1.0) == 1.0) + #expect(CameraController.clampQuality(1.1) == 1.0) + } + + @Test func clampDurationDefaultsAndBounds() { + #expect(CameraController.clampDurationMs(nil) == 3000) + #expect(CameraController.clampDurationMs(0) == 250) + #expect(CameraController.clampDurationMs(249) == 250) + #expect(CameraController.clampDurationMs(250) == 250) + #expect(CameraController.clampDurationMs(1000) == 1000) + #expect(CameraController.clampDurationMs(60000) == 60000) + #expect(CameraController.clampDurationMs(60001) == 60000) + } +} diff --git a/apps/ios/Tests/CameraControllerErrorTests.swift b/apps/ios/Tests/CameraControllerErrorTests.swift new file mode 100644 index 0000000000000000000000000000000000000000..26cac6177daa36691f904b845d03bea640e39689 --- /dev/null +++ b/apps/ios/Tests/CameraControllerErrorTests.swift @@ -0,0 +1,14 @@ +import Testing +@testable import OpenClaw + +@Suite struct CameraControllerErrorTests { + @Test func errorDescriptionsAreStable() { + #expect(CameraController.CameraError.cameraUnavailable.errorDescription == "Camera unavailable") + #expect(CameraController.CameraError.microphoneUnavailable.errorDescription == "Microphone unavailable") + #expect(CameraController.CameraError.permissionDenied(kind: "Camera") + .errorDescription == "Camera permission denied") + #expect(CameraController.CameraError.invalidParams("bad").errorDescription == "bad") + #expect(CameraController.CameraError.captureFailed("nope").errorDescription == "nope") + #expect(CameraController.CameraError.exportFailed("export").errorDescription == "export") + } +} diff --git a/apps/ios/Tests/DeepLinkParserTests.swift b/apps/ios/Tests/DeepLinkParserTests.swift new file mode 100644 index 0000000000000000000000000000000000000000..9a3d8618738707e505b58c79e79f4630759518ca --- /dev/null +++ b/apps/ios/Tests/DeepLinkParserTests.swift @@ -0,0 +1,79 @@ +import OpenClawKit +import Foundation +import Testing + +@Suite struct DeepLinkParserTests { + @Test func parseRejectsUnknownHost() { + let url = URL(string: "openclaw://nope?message=hi")! + #expect(DeepLinkParser.parse(url) == nil) + } + + @Test func parseHostIsCaseInsensitive() { + let url = URL(string: "openclaw://AGENT?message=Hello")! + #expect(DeepLinkParser.parse(url) == .agent(.init( + message: "Hello", + sessionKey: nil, + thinking: nil, + deliver: false, + to: nil, + channel: nil, + timeoutSeconds: nil, + key: nil))) + } + + @Test func parseRejectsNonOpenClawScheme() { + let url = URL(string: "https://example.com/agent?message=hi")! + #expect(DeepLinkParser.parse(url) == nil) + } + + @Test func parseRejectsEmptyMessage() { + let url = URL(string: "openclaw://agent?message=%20%20%0A")! + #expect(DeepLinkParser.parse(url) == nil) + } + + @Test func parseAgentLinkParsesCommonFields() { + let url = + URL(string: "openclaw://agent?message=Hello&deliver=1&sessionKey=node-test&thinking=low&timeoutSeconds=30")! + #expect( + DeepLinkParser.parse(url) == .agent( + .init( + message: "Hello", + sessionKey: "node-test", + thinking: "low", + deliver: true, + to: nil, + channel: nil, + timeoutSeconds: 30, + key: nil))) + } + + @Test func parseAgentLinkParsesTargetRoutingFields() { + let url = + URL( + string: "openclaw://agent?message=Hello%20World&deliver=1&to=%2B15551234567&channel=whatsapp&key=secret")! + #expect( + DeepLinkParser.parse(url) == .agent( + .init( + message: "Hello World", + sessionKey: nil, + thinking: nil, + deliver: true, + to: "+15551234567", + channel: "whatsapp", + timeoutSeconds: nil, + key: "secret"))) + } + + @Test func parseRejectsNegativeTimeoutSeconds() { + let url = URL(string: "openclaw://agent?message=Hello&timeoutSeconds=-1")! + #expect(DeepLinkParser.parse(url) == .agent(.init( + message: "Hello", + sessionKey: nil, + thinking: nil, + deliver: false, + to: nil, + channel: nil, + timeoutSeconds: nil, + key: nil))) + } +} diff --git a/apps/ios/Tests/GatewayConnectionControllerTests.swift b/apps/ios/Tests/GatewayConnectionControllerTests.swift new file mode 100644 index 0000000000000000000000000000000000000000..0d3bdbba0ee57fc29ed4c43850d919e5193278b5 --- /dev/null +++ b/apps/ios/Tests/GatewayConnectionControllerTests.swift @@ -0,0 +1,79 @@ +import OpenClawKit +import Foundation +import Testing +import UIKit +@testable import OpenClaw + +private func withUserDefaults(_ updates: [String: Any?], _ body: () throws -> T) rethrows -> T { + let defaults = UserDefaults.standard + var snapshot: [String: Any?] = [:] + for key in updates.keys { + snapshot[key] = defaults.object(forKey: key) + } + for (key, value) in updates { + if let value { + defaults.set(value, forKey: key) + } else { + defaults.removeObject(forKey: key) + } + } + defer { + for (key, value) in snapshot { + if let value { + defaults.set(value, forKey: key) + } else { + defaults.removeObject(forKey: key) + } + } + } + return try body() +} + +@Suite(.serialized) struct GatewayConnectionControllerTests { + @Test @MainActor func resolvedDisplayNameSetsDefaultWhenMissing() { + let defaults = UserDefaults.standard + let displayKey = "node.displayName" + + withUserDefaults([displayKey: nil, "node.instanceId": "ios-test"]) { + let appModel = NodeAppModel() + let controller = GatewayConnectionController(appModel: appModel, startDiscovery: false) + + let resolved = controller._test_resolvedDisplayName(defaults: defaults) + #expect(!resolved.isEmpty) + #expect(defaults.string(forKey: displayKey) == resolved) + } + } + + @Test @MainActor func currentCapsReflectToggles() { + withUserDefaults([ + "node.instanceId": "ios-test", + "node.displayName": "Test Node", + "camera.enabled": true, + "location.enabledMode": OpenClawLocationMode.always.rawValue, + VoiceWakePreferences.enabledKey: true, + ]) { + let appModel = NodeAppModel() + let controller = GatewayConnectionController(appModel: appModel, startDiscovery: false) + let caps = Set(controller._test_currentCaps()) + + #expect(caps.contains(OpenClawCapability.canvas.rawValue)) + #expect(caps.contains(OpenClawCapability.screen.rawValue)) + #expect(caps.contains(OpenClawCapability.camera.rawValue)) + #expect(caps.contains(OpenClawCapability.location.rawValue)) + #expect(caps.contains(OpenClawCapability.voiceWake.rawValue)) + } + } + + @Test @MainActor func currentCommandsIncludeLocationWhenEnabled() { + withUserDefaults([ + "node.instanceId": "ios-test", + "location.enabledMode": OpenClawLocationMode.whileUsing.rawValue, + ]) { + let appModel = NodeAppModel() + let controller = GatewayConnectionController(appModel: appModel, startDiscovery: false) + let commands = Set(controller._test_currentCommands()) + + #expect(commands.contains(OpenClawLocationCommand.get.rawValue)) + } + } +} diff --git a/apps/ios/Tests/GatewayDiscoveryModelTests.swift b/apps/ios/Tests/GatewayDiscoveryModelTests.swift new file mode 100644 index 0000000000000000000000000000000000000000..2f98948c962dc8eacec942f7b4eb26e56c20a1e4 --- /dev/null +++ b/apps/ios/Tests/GatewayDiscoveryModelTests.swift @@ -0,0 +1,22 @@ +import Testing +@testable import OpenClaw + +@Suite(.serialized) struct GatewayDiscoveryModelTests { + @Test @MainActor func debugLoggingCapturesLifecycleAndResets() { + let model = GatewayDiscoveryModel() + + #expect(model.debugLog.isEmpty) + #expect(model.statusText == "Idle") + + model.setDebugLoggingEnabled(true) + #expect(model.debugLog.count >= 2) + + model.stop() + #expect(model.statusText == "Stopped") + #expect(model.gateways.isEmpty) + #expect(model.debugLog.count >= 3) + + model.setDebugLoggingEnabled(false) + #expect(model.debugLog.isEmpty) + } +} diff --git a/apps/ios/Tests/GatewayEndpointIDTests.swift b/apps/ios/Tests/GatewayEndpointIDTests.swift new file mode 100644 index 0000000000000000000000000000000000000000..e6edf2df237641f793e470f9db3b92c35d73a9cb --- /dev/null +++ b/apps/ios/Tests/GatewayEndpointIDTests.swift @@ -0,0 +1,33 @@ +import OpenClawKit +import Network +import Testing +@testable import OpenClaw + +@Suite struct GatewayEndpointIDTests { + @Test func stableIDForServiceDecodesAndNormalizesName() { + let endpoint = NWEndpoint.service( + name: "OpenClaw\\032Gateway \\032 Node\n", + type: "_openclaw-gw._tcp", + domain: "local.", + interface: nil) + + #expect(GatewayEndpointID.stableID(endpoint) == "_openclaw-gw._tcp|local.|OpenClaw Gateway Node") + } + + @Test func stableIDForNonServiceUsesEndpointDescription() { + let endpoint = NWEndpoint.hostPort(host: NWEndpoint.Host("127.0.0.1"), port: 4242) + #expect(GatewayEndpointID.stableID(endpoint) == String(describing: endpoint)) + } + + @Test func prettyDescriptionDecodesBonjourEscapes() { + let endpoint = NWEndpoint.service( + name: "OpenClaw\\032Gateway", + type: "_openclaw-gw._tcp", + domain: "local.", + interface: nil) + + let pretty = GatewayEndpointID.prettyDescription(endpoint) + #expect(pretty == BonjourEscapes.decode(String(describing: endpoint))) + #expect(!pretty.localizedCaseInsensitiveContains("\\032")) + } +} diff --git a/apps/ios/Tests/GatewaySettingsStoreTests.swift b/apps/ios/Tests/GatewaySettingsStoreTests.swift new file mode 100644 index 0000000000000000000000000000000000000000..255c7aac9b2d9a7b615dcb9fbebe887c6edcede7 --- /dev/null +++ b/apps/ios/Tests/GatewaySettingsStoreTests.swift @@ -0,0 +1,127 @@ +import Foundation +import Testing +@testable import OpenClaw + +private struct KeychainEntry: Hashable { + let service: String + let account: String +} + +private let gatewayService = "bot.molt.gateway" +private let nodeService = "bot.molt.node" +private let instanceIdEntry = KeychainEntry(service: nodeService, account: "instanceId") +private let preferredGatewayEntry = KeychainEntry(service: gatewayService, account: "preferredStableID") +private let lastGatewayEntry = KeychainEntry(service: gatewayService, account: "lastDiscoveredStableID") + +private func snapshotDefaults(_ keys: [String]) -> [String: Any?] { + let defaults = UserDefaults.standard + var snapshot: [String: Any?] = [:] + for key in keys { + snapshot[key] = defaults.object(forKey: key) + } + return snapshot +} + +private func applyDefaults(_ values: [String: Any?]) { + let defaults = UserDefaults.standard + for (key, value) in values { + if let value { + defaults.set(value, forKey: key) + } else { + defaults.removeObject(forKey: key) + } + } +} + +private func restoreDefaults(_ snapshot: [String: Any?]) { + applyDefaults(snapshot) +} + +private func snapshotKeychain(_ entries: [KeychainEntry]) -> [KeychainEntry: String?] { + var snapshot: [KeychainEntry: String?] = [:] + for entry in entries { + snapshot[entry] = KeychainStore.loadString(service: entry.service, account: entry.account) + } + return snapshot +} + +private func applyKeychain(_ values: [KeychainEntry: String?]) { + for (entry, value) in values { + if let value { + _ = KeychainStore.saveString(value, service: entry.service, account: entry.account) + } else { + _ = KeychainStore.delete(service: entry.service, account: entry.account) + } + } +} + +private func restoreKeychain(_ snapshot: [KeychainEntry: String?]) { + applyKeychain(snapshot) +} + +@Suite(.serialized) struct GatewaySettingsStoreTests { + @Test func bootstrapCopiesDefaultsToKeychainWhenMissing() { + let defaultsKeys = [ + "node.instanceId", + "gateway.preferredStableID", + "gateway.lastDiscoveredStableID", + ] + let entries = [instanceIdEntry, preferredGatewayEntry, lastGatewayEntry] + let defaultsSnapshot = snapshotDefaults(defaultsKeys) + let keychainSnapshot = snapshotKeychain(entries) + defer { + restoreDefaults(defaultsSnapshot) + restoreKeychain(keychainSnapshot) + } + + applyDefaults([ + "node.instanceId": "node-test", + "gateway.preferredStableID": "preferred-test", + "gateway.lastDiscoveredStableID": "last-test", + ]) + applyKeychain([ + instanceIdEntry: nil, + preferredGatewayEntry: nil, + lastGatewayEntry: nil, + ]) + + GatewaySettingsStore.bootstrapPersistence() + + #expect(KeychainStore.loadString(service: nodeService, account: "instanceId") == "node-test") + #expect(KeychainStore.loadString(service: gatewayService, account: "preferredStableID") == "preferred-test") + #expect(KeychainStore.loadString(service: gatewayService, account: "lastDiscoveredStableID") == "last-test") + } + + @Test func bootstrapCopiesKeychainToDefaultsWhenMissing() { + let defaultsKeys = [ + "node.instanceId", + "gateway.preferredStableID", + "gateway.lastDiscoveredStableID", + ] + let entries = [instanceIdEntry, preferredGatewayEntry, lastGatewayEntry] + let defaultsSnapshot = snapshotDefaults(defaultsKeys) + let keychainSnapshot = snapshotKeychain(entries) + defer { + restoreDefaults(defaultsSnapshot) + restoreKeychain(keychainSnapshot) + } + + applyDefaults([ + "node.instanceId": nil, + "gateway.preferredStableID": nil, + "gateway.lastDiscoveredStableID": nil, + ]) + applyKeychain([ + instanceIdEntry: "node-from-keychain", + preferredGatewayEntry: "preferred-from-keychain", + lastGatewayEntry: "last-from-keychain", + ]) + + GatewaySettingsStore.bootstrapPersistence() + + let defaults = UserDefaults.standard + #expect(defaults.string(forKey: "node.instanceId") == "node-from-keychain") + #expect(defaults.string(forKey: "gateway.preferredStableID") == "preferred-from-keychain") + #expect(defaults.string(forKey: "gateway.lastDiscoveredStableID") == "last-from-keychain") + } +} diff --git a/apps/ios/Tests/IOSGatewayChatTransportTests.swift b/apps/ios/Tests/IOSGatewayChatTransportTests.swift new file mode 100644 index 0000000000000000000000000000000000000000..f49f242ff247694e8d3da07d943422902e04af2d --- /dev/null +++ b/apps/ios/Tests/IOSGatewayChatTransportTests.swift @@ -0,0 +1,30 @@ +import OpenClawKit +import Testing +@testable import OpenClaw + +@Suite struct IOSGatewayChatTransportTests { + @Test func requestsFailFastWhenGatewayNotConnected() async { + let gateway = GatewayNodeSession() + let transport = IOSGatewayChatTransport(gateway: gateway) + + do { + _ = try await transport.requestHistory(sessionKey: "node-test") + Issue.record("Expected requestHistory to throw when gateway not connected") + } catch {} + + do { + _ = try await transport.sendMessage( + sessionKey: "node-test", + message: "hello", + thinking: "low", + idempotencyKey: "idempotency", + attachments: []) + Issue.record("Expected sendMessage to throw when gateway not connected") + } catch {} + + do { + _ = try await transport.requestHealth(timeoutMs: 250) + Issue.record("Expected requestHealth to throw when gateway not connected") + } catch {} + } +} diff --git a/apps/ios/Tests/Info.plist b/apps/ios/Tests/Info.plist new file mode 100644 index 0000000000000000000000000000000000000000..8a57da2916106057a856b393ff53add02192ab50 --- /dev/null +++ b/apps/ios/Tests/Info.plist @@ -0,0 +1,24 @@ + + + + + CFBundleDevelopmentRegion + $(DEVELOPMENT_LANGUAGE) + CFBundleDisplayName + OpenClawTests + CFBundleExecutable + $(EXECUTABLE_NAME) + CFBundleIdentifier + $(PRODUCT_BUNDLE_IDENTIFIER) + CFBundleInfoDictionaryVersion + 6.0 + CFBundleName + $(PRODUCT_NAME) + CFBundlePackageType + BNDL + CFBundleShortVersionString + 2026.1.30 + CFBundleVersion + 20260129 + + diff --git a/apps/ios/Tests/KeychainStoreTests.swift b/apps/ios/Tests/KeychainStoreTests.swift new file mode 100644 index 0000000000000000000000000000000000000000..827be250ed7fce20702806d31f2a139d3fc614ee --- /dev/null +++ b/apps/ios/Tests/KeychainStoreTests.swift @@ -0,0 +1,22 @@ +import Foundation +import Testing +@testable import OpenClaw + +@Suite struct KeychainStoreTests { + @Test func saveLoadUpdateDeleteRoundTrip() { + let service = "bot.molt.tests.\(UUID().uuidString)" + let account = "value" + + #expect(KeychainStore.delete(service: service, account: account)) + #expect(KeychainStore.loadString(service: service, account: account) == nil) + + #expect(KeychainStore.saveString("first", service: service, account: account)) + #expect(KeychainStore.loadString(service: service, account: account) == "first") + + #expect(KeychainStore.saveString("second", service: service, account: account)) + #expect(KeychainStore.loadString(service: service, account: account) == "second") + + #expect(KeychainStore.delete(service: service, account: account)) + #expect(KeychainStore.loadString(service: service, account: account) == nil) + } +} diff --git a/apps/ios/Tests/NodeAppModelInvokeTests.swift b/apps/ios/Tests/NodeAppModelInvokeTests.swift new file mode 100644 index 0000000000000000000000000000000000000000..124059021d67d12586291b20552dd19995e443d0 --- /dev/null +++ b/apps/ios/Tests/NodeAppModelInvokeTests.swift @@ -0,0 +1,194 @@ +import OpenClawKit +import Foundation +import Testing +import UIKit +@testable import OpenClaw + +private func withUserDefaults(_ updates: [String: Any?], _ body: () throws -> T) rethrows -> T { + let defaults = UserDefaults.standard + var snapshot: [String: Any?] = [:] + for key in updates.keys { + snapshot[key] = defaults.object(forKey: key) + } + for (key, value) in updates { + if let value { + defaults.set(value, forKey: key) + } else { + defaults.removeObject(forKey: key) + } + } + defer { + for (key, value) in snapshot { + if let value { + defaults.set(value, forKey: key) + } else { + defaults.removeObject(forKey: key) + } + } + } + return try body() +} + +@Suite(.serialized) struct NodeAppModelInvokeTests { + @Test @MainActor func decodeParamsFailsWithoutJSON() { + #expect(throws: Error.self) { + _ = try NodeAppModel._test_decodeParams(OpenClawCanvasNavigateParams.self, from: nil) + } + } + + @Test @MainActor func encodePayloadEmitsJSON() throws { + struct Payload: Codable, Equatable { + var value: String + } + let json = try NodeAppModel._test_encodePayload(Payload(value: "ok")) + #expect(json.contains("\"value\"")) + } + + @Test @MainActor func handleInvokeRejectsBackgroundCommands() async { + let appModel = NodeAppModel() + appModel.setScenePhase(.background) + + let req = BridgeInvokeRequest(id: "bg", command: OpenClawCanvasCommand.present.rawValue) + let res = await appModel._test_handleInvoke(req) + #expect(res.ok == false) + #expect(res.error?.code == .backgroundUnavailable) + } + + @Test @MainActor func handleInvokeRejectsCameraWhenDisabled() async { + let appModel = NodeAppModel() + let req = BridgeInvokeRequest(id: "cam", command: OpenClawCameraCommand.snap.rawValue) + + let defaults = UserDefaults.standard + let key = "camera.enabled" + let previous = defaults.object(forKey: key) + defaults.set(false, forKey: key) + defer { + if let previous { + defaults.set(previous, forKey: key) + } else { + defaults.removeObject(forKey: key) + } + } + + let res = await appModel._test_handleInvoke(req) + #expect(res.ok == false) + #expect(res.error?.code == .unavailable) + #expect(res.error?.message.contains("CAMERA_DISABLED") == true) + } + + @Test @MainActor func handleInvokeRejectsInvalidScreenFormat() async { + let appModel = NodeAppModel() + let params = OpenClawScreenRecordParams(format: "gif") + let data = try? JSONEncoder().encode(params) + let json = data.flatMap { String(data: $0, encoding: .utf8) } + + let req = BridgeInvokeRequest( + id: "screen", + command: OpenClawScreenCommand.record.rawValue, + paramsJSON: json) + + let res = await appModel._test_handleInvoke(req) + #expect(res.ok == false) + #expect(res.error?.message.contains("screen format must be mp4") == true) + } + + @Test @MainActor func handleInvokeCanvasCommandsUpdateScreen() async throws { + let appModel = NodeAppModel() + appModel.screen.navigate(to: "http://example.com") + + let present = BridgeInvokeRequest(id: "present", command: OpenClawCanvasCommand.present.rawValue) + let presentRes = await appModel._test_handleInvoke(present) + #expect(presentRes.ok == true) + #expect(appModel.screen.urlString.isEmpty) + + let navigateParams = OpenClawCanvasNavigateParams(url: "http://localhost:18789/") + let navData = try JSONEncoder().encode(navigateParams) + let navJSON = String(decoding: navData, as: UTF8.self) + let navigate = BridgeInvokeRequest( + id: "nav", + command: OpenClawCanvasCommand.navigate.rawValue, + paramsJSON: navJSON) + let navRes = await appModel._test_handleInvoke(navigate) + #expect(navRes.ok == true) + #expect(appModel.screen.urlString == "http://localhost:18789/") + + let evalParams = OpenClawCanvasEvalParams(javaScript: "1+1") + let evalData = try JSONEncoder().encode(evalParams) + let evalJSON = String(decoding: evalData, as: UTF8.self) + let eval = BridgeInvokeRequest( + id: "eval", + command: OpenClawCanvasCommand.evalJS.rawValue, + paramsJSON: evalJSON) + let evalRes = await appModel._test_handleInvoke(eval) + #expect(evalRes.ok == true) + let payloadData = try #require(evalRes.payloadJSON?.data(using: .utf8)) + let payload = try JSONSerialization.jsonObject(with: payloadData) as? [String: Any] + #expect(payload?["result"] as? String == "2") + } + + @Test @MainActor func handleInvokeA2UICommandsFailWhenHostMissing() async throws { + let appModel = NodeAppModel() + + let reset = BridgeInvokeRequest(id: "reset", command: OpenClawCanvasA2UICommand.reset.rawValue) + let resetRes = await appModel._test_handleInvoke(reset) + #expect(resetRes.ok == false) + #expect(resetRes.error?.message.contains("A2UI_HOST_NOT_CONFIGURED") == true) + + let jsonl = "{\"beginRendering\":{}}" + let pushParams = OpenClawCanvasA2UIPushJSONLParams(jsonl: jsonl) + let pushData = try JSONEncoder().encode(pushParams) + let pushJSON = String(decoding: pushData, as: UTF8.self) + let push = BridgeInvokeRequest( + id: "push", + command: OpenClawCanvasA2UICommand.pushJSONL.rawValue, + paramsJSON: pushJSON) + let pushRes = await appModel._test_handleInvoke(push) + #expect(pushRes.ok == false) + #expect(pushRes.error?.message.contains("A2UI_HOST_NOT_CONFIGURED") == true) + } + + @Test @MainActor func handleInvokeUnknownCommandReturnsInvalidRequest() async { + let appModel = NodeAppModel() + let req = BridgeInvokeRequest(id: "unknown", command: "nope") + let res = await appModel._test_handleInvoke(req) + #expect(res.ok == false) + #expect(res.error?.code == .invalidRequest) + } + + @Test @MainActor func handleDeepLinkSetsErrorWhenNotConnected() async { + let appModel = NodeAppModel() + let url = URL(string: "openclaw://agent?message=hello")! + await appModel.handleDeepLink(url: url) + #expect(appModel.screen.errorText?.contains("Gateway not connected") == true) + } + + @Test @MainActor func handleDeepLinkRejectsOversizedMessage() async { + let appModel = NodeAppModel() + let msg = String(repeating: "a", count: 20001) + let url = URL(string: "openclaw://agent?message=\(msg)")! + await appModel.handleDeepLink(url: url) + #expect(appModel.screen.errorText?.contains("Deep link too large") == true) + } + + @Test @MainActor func sendVoiceTranscriptThrowsWhenGatewayOffline() async { + let appModel = NodeAppModel() + await #expect(throws: Error.self) { + try await appModel.sendVoiceTranscript(text: "hello", sessionKey: "main") + } + } + + @Test @MainActor func canvasA2UIActionDispatchesStatus() async { + let appModel = NodeAppModel() + let body: [String: Any] = [ + "userAction": [ + "name": "tap", + "id": "action-1", + "surfaceId": "main", + "sourceComponentId": "button-1", + "context": ["value": "ok"], + ], + ] + await appModel._test_handleCanvasA2UIAction(body: body) + #expect(appModel.screen.urlString.isEmpty) + } +} diff --git a/apps/ios/Tests/ScreenControllerTests.swift b/apps/ios/Tests/ScreenControllerTests.swift new file mode 100644 index 0000000000000000000000000000000000000000..32c36acacb7b14f7dc4e5e27245ca523ba96de98 --- /dev/null +++ b/apps/ios/Tests/ScreenControllerTests.swift @@ -0,0 +1,71 @@ +import Testing +import WebKit +@testable import OpenClaw + +@Suite struct ScreenControllerTests { + @Test @MainActor func canvasModeConfiguresWebViewForTouch() { + let screen = ScreenController() + + #expect(screen.webView.isOpaque == true) + #expect(screen.webView.backgroundColor == .black) + + let scrollView = screen.webView.scrollView + #expect(scrollView.backgroundColor == .black) + #expect(scrollView.contentInsetAdjustmentBehavior == .never) + #expect(scrollView.isScrollEnabled == false) + #expect(scrollView.bounces == false) + } + + @Test @MainActor func navigateEnablesScrollForWebPages() { + let screen = ScreenController() + screen.navigate(to: "https://example.com") + + let scrollView = screen.webView.scrollView + #expect(scrollView.isScrollEnabled == true) + #expect(scrollView.bounces == true) + } + + @Test @MainActor func navigateSlashShowsDefaultCanvas() { + let screen = ScreenController() + screen.navigate(to: "/") + + #expect(screen.urlString.isEmpty) + } + + @Test @MainActor func evalExecutesJavaScript() async throws { + let screen = ScreenController() + let deadline = ContinuousClock().now.advanced(by: .seconds(3)) + + while true { + do { + let result = try await screen.eval(javaScript: "1+1") + #expect(result == "2") + return + } catch { + if ContinuousClock().now >= deadline { + throw error + } + try? await Task.sleep(nanoseconds: 100_000_000) + } + } + } + + @Test @MainActor func localNetworkCanvasURLsAreAllowed() { + let screen = ScreenController() + #expect(screen.isLocalNetworkCanvasURL(URL(string: "http://localhost:18789/")!) == true) + #expect(screen.isLocalNetworkCanvasURL(URL(string: "http://openclaw.local:18789/")!) == true) + #expect(screen.isLocalNetworkCanvasURL(URL(string: "http://peters-mac-studio-1:18789/")!) == true) + #expect(screen.isLocalNetworkCanvasURL(URL(string: "https://peters-mac-studio-1.ts.net:18789/")!) == true) + #expect(screen.isLocalNetworkCanvasURL(URL(string: "http://192.168.0.10:18789/")!) == true) + #expect(screen.isLocalNetworkCanvasURL(URL(string: "http://10.0.0.10:18789/")!) == true) + #expect(screen.isLocalNetworkCanvasURL(URL(string: "http://100.123.224.76:18789/")!) == true) // Tailscale CGNAT + #expect(screen.isLocalNetworkCanvasURL(URL(string: "https://example.com/")!) == false) + #expect(screen.isLocalNetworkCanvasURL(URL(string: "http://8.8.8.8/")!) == false) + } + + @Test func parseA2UIActionBodyAcceptsJSONString() throws { + let body = ScreenController.parseA2UIActionBody("{\"userAction\":{\"name\":\"hello\"}}") + let userAction = try #require(body?["userAction"] as? [String: Any]) + #expect(userAction["name"] as? String == "hello") + } +} diff --git a/apps/ios/Tests/ScreenRecordServiceTests.swift b/apps/ios/Tests/ScreenRecordServiceTests.swift new file mode 100644 index 0000000000000000000000000000000000000000..6ae8f1ca30f740f25b63245dc967216284ca6916 --- /dev/null +++ b/apps/ios/Tests/ScreenRecordServiceTests.swift @@ -0,0 +1,32 @@ +import Testing +@testable import OpenClaw + +@Suite(.serialized) struct ScreenRecordServiceTests { + @Test func clampDefaultsAndBounds() { + #expect(ScreenRecordService._test_clampDurationMs(nil) == 10000) + #expect(ScreenRecordService._test_clampDurationMs(0) == 250) + #expect(ScreenRecordService._test_clampDurationMs(60001) == 60000) + + #expect(ScreenRecordService._test_clampFps(nil) == 10) + #expect(ScreenRecordService._test_clampFps(0) == 1) + #expect(ScreenRecordService._test_clampFps(120) == 30) + #expect(ScreenRecordService._test_clampFps(.infinity) == 10) + } + + @Test @MainActor func recordRejectsInvalidScreenIndex() async { + let recorder = ScreenRecordService() + do { + _ = try await recorder.record( + screenIndex: 1, + durationMs: 250, + fps: 5, + includeAudio: false, + outPath: nil) + Issue.record("Expected invalid screen index to throw") + } catch let error as ScreenRecordService.ScreenRecordError { + #expect(error.localizedDescription.contains("Invalid screen index") == true) + } catch { + Issue.record("Unexpected error type: \(error)") + } + } +} diff --git a/apps/ios/Tests/SettingsNetworkingHelpersTests.swift b/apps/ios/Tests/SettingsNetworkingHelpersTests.swift new file mode 100644 index 0000000000000000000000000000000000000000..f1a649613b58596eb6f622c166947dcb70191ca4 --- /dev/null +++ b/apps/ios/Tests/SettingsNetworkingHelpersTests.swift @@ -0,0 +1,50 @@ +import Testing +@testable import OpenClaw + +@Suite struct SettingsNetworkingHelpersTests { + @Test func parseHostPortParsesIPv4() { + #expect(SettingsNetworkingHelpers.parseHostPort(from: "127.0.0.1:8080") == .init(host: "127.0.0.1", port: 8080)) + } + + @Test func parseHostPortParsesHostnameAndTrims() { + #expect(SettingsNetworkingHelpers.parseHostPort(from: " example.com:80 \n") == .init( + host: "example.com", + port: 80)) + } + + @Test func parseHostPortParsesBracketedIPv6() { + #expect( + SettingsNetworkingHelpers.parseHostPort(from: "[2001:db8::1]:443") == + .init(host: "2001:db8::1", port: 443)) + } + + @Test func parseHostPortRejectsMissingPort() { + #expect(SettingsNetworkingHelpers.parseHostPort(from: "example.com") == nil) + #expect(SettingsNetworkingHelpers.parseHostPort(from: "[2001:db8::1]") == nil) + } + + @Test func parseHostPortRejectsInvalidPort() { + #expect(SettingsNetworkingHelpers.parseHostPort(from: "example.com:lol") == nil) + #expect(SettingsNetworkingHelpers.parseHostPort(from: "[2001:db8::1]:lol") == nil) + } + + @Test func httpURLStringFormatsIPv4AndPort() { + #expect(SettingsNetworkingHelpers + .httpURLString(host: "127.0.0.1", port: 8080, fallback: "fallback") == "http://127.0.0.1:8080") + } + + @Test func httpURLStringBracketsIPv6() { + #expect(SettingsNetworkingHelpers + .httpURLString(host: "2001:db8::1", port: 8080, fallback: "fallback") == "http://[2001:db8::1]:8080") + } + + @Test func httpURLStringLeavesAlreadyBracketedIPv6() { + #expect(SettingsNetworkingHelpers + .httpURLString(host: "[2001:db8::1]", port: 8080, fallback: "fallback") == "http://[2001:db8::1]:8080") + } + + @Test func httpURLStringFallsBackWhenMissingHostOrPort() { + #expect(SettingsNetworkingHelpers.httpURLString(host: nil, port: 80, fallback: "x") == "http://x") + #expect(SettingsNetworkingHelpers.httpURLString(host: "example.com", port: nil, fallback: "y") == "http://y") + } +} diff --git a/apps/ios/Tests/SwiftUIRenderSmokeTests.swift b/apps/ios/Tests/SwiftUIRenderSmokeTests.swift new file mode 100644 index 0000000000000000000000000000000000000000..4e13b3f4cd172cca460c8074dfda5126016f6fec --- /dev/null +++ b/apps/ios/Tests/SwiftUIRenderSmokeTests.swift @@ -0,0 +1,81 @@ +import OpenClawKit +import SwiftUI +import Testing +import UIKit +@testable import OpenClaw + +@Suite struct SwiftUIRenderSmokeTests { + @MainActor private static func host(_ view: some View) -> UIWindow { + let window = UIWindow(frame: UIScreen.main.bounds) + window.rootViewController = UIHostingController(rootView: view) + window.makeKeyAndVisible() + window.rootViewController?.view.setNeedsLayout() + window.rootViewController?.view.layoutIfNeeded() + return window + } + + @Test @MainActor func statusPillConnectingBuildsAViewHierarchy() { + let root = StatusPill(gateway: .connecting, voiceWakeEnabled: true, brighten: true) {} + _ = Self.host(root) + } + + @Test @MainActor func statusPillDisconnectedBuildsAViewHierarchy() { + let root = StatusPill(gateway: .disconnected, voiceWakeEnabled: false) {} + _ = Self.host(root) + } + + @Test @MainActor func settingsTabBuildsAViewHierarchy() { + let appModel = NodeAppModel() + let gatewayController = GatewayConnectionController(appModel: appModel, startDiscovery: false) + + let root = SettingsTab() + .environment(appModel) + .environment(appModel.voiceWake) + .environment(gatewayController) + + _ = Self.host(root) + } + + @Test @MainActor func rootTabsBuildAViewHierarchy() { + let appModel = NodeAppModel() + let gatewayController = GatewayConnectionController(appModel: appModel, startDiscovery: false) + + let root = RootTabs() + .environment(appModel) + .environment(appModel.voiceWake) + .environment(gatewayController) + + _ = Self.host(root) + } + + @Test @MainActor func voiceTabBuildsAViewHierarchy() { + let appModel = NodeAppModel() + + let root = VoiceTab() + .environment(appModel) + .environment(appModel.voiceWake) + + _ = Self.host(root) + } + + @Test @MainActor func voiceWakeWordsViewBuildsAViewHierarchy() { + let appModel = NodeAppModel() + let root = NavigationStack { VoiceWakeWordsSettingsView() } + .environment(appModel) + _ = Self.host(root) + } + + @Test @MainActor func chatSheetBuildsAViewHierarchy() { + let appModel = NodeAppModel() + let gateway = GatewayNodeSession() + let root = ChatSheet(gateway: gateway, sessionKey: "test") + .environment(appModel) + .environment(appModel.voiceWake) + _ = Self.host(root) + } + + @Test @MainActor func voiceWakeToastBuildsAViewHierarchy() { + let root = VoiceWakeToast(command: "openclaw: do something") + _ = Self.host(root) + } +} diff --git a/apps/ios/Tests/VoiceWakeGatewaySyncTests.swift b/apps/ios/Tests/VoiceWakeGatewaySyncTests.swift new file mode 100644 index 0000000000000000000000000000000000000000..fa4a070da289498940751d05b6a41f8bd432bac3 --- /dev/null +++ b/apps/ios/Tests/VoiceWakeGatewaySyncTests.swift @@ -0,0 +1,22 @@ +import Foundation +import Testing +@testable import OpenClaw + +@Suite struct VoiceWakeGatewaySyncTests { + @Test func decodeGatewayTriggersFromJSONSanitizes() { + let payload = #"{"triggers":[" openclaw ","", "computer"]}"# + let triggers = VoiceWakePreferences.decodeGatewayTriggers(from: payload) + #expect(triggers == ["openclaw", "computer"]) + } + + @Test func decodeGatewayTriggersFromJSONFallsBackWhenEmpty() { + let payload = #"{"triggers":[" ",""]}"# + let triggers = VoiceWakePreferences.decodeGatewayTriggers(from: payload) + #expect(triggers == VoiceWakePreferences.defaultTriggerWords) + } + + @Test func decodeGatewayTriggersFromInvalidJSONReturnsNil() { + let triggers = VoiceWakePreferences.decodeGatewayTriggers(from: "not json") + #expect(triggers == nil) + } +} diff --git a/apps/ios/Tests/VoiceWakeManagerExtractCommandTests.swift b/apps/ios/Tests/VoiceWakeManagerExtractCommandTests.swift new file mode 100644 index 0000000000000000000000000000000000000000..f6b0378cd6bd766a25034bc5e2f4e924cc54c80e --- /dev/null +++ b/apps/ios/Tests/VoiceWakeManagerExtractCommandTests.swift @@ -0,0 +1,90 @@ +import Foundation +import SwabbleKit +import Testing +@testable import OpenClaw + +@Suite struct VoiceWakeManagerExtractCommandTests { + @Test func extractCommandReturnsNilWhenNoTriggerFound() { + let transcript = "hello world" + let segments = makeSegments( + transcript: transcript, + words: [("hello", 0.0, 0.1), ("world", 0.2, 0.1)]) + #expect(VoiceWakeManager.extractCommand(from: transcript, segments: segments, triggers: ["openclaw"]) == nil) + } + + @Test func extractCommandTrimsTokensAndResult() { + let transcript = "hey openclaw do thing" + let segments = makeSegments( + transcript: transcript, + words: [ + ("hey", 0.0, 0.1), + ("openclaw", 0.2, 0.1), + ("do", 0.9, 0.1), + ("thing", 1.1, 0.1), + ]) + let cmd = VoiceWakeManager.extractCommand( + from: transcript, + segments: segments, + triggers: [" openclaw "], + minPostTriggerGap: 0.3) + #expect(cmd == "do thing") + } + + @Test func extractCommandReturnsNilWhenGapTooShort() { + let transcript = "hey openclaw do thing" + let segments = makeSegments( + transcript: transcript, + words: [ + ("hey", 0.0, 0.1), + ("openclaw", 0.2, 0.1), + ("do", 0.35, 0.1), + ("thing", 0.5, 0.1), + ]) + let cmd = VoiceWakeManager.extractCommand( + from: transcript, + segments: segments, + triggers: ["openclaw"], + minPostTriggerGap: 0.3) + #expect(cmd == nil) + } + + @Test func extractCommandReturnsNilWhenNothingAfterTrigger() { + let transcript = "hey openclaw" + let segments = makeSegments( + transcript: transcript, + words: [("hey", 0.0, 0.1), ("openclaw", 0.2, 0.1)]) + #expect(VoiceWakeManager.extractCommand(from: transcript, segments: segments, triggers: ["openclaw"]) == nil) + } + + @Test func extractCommandIgnoresEmptyTriggers() { + let transcript = "hey openclaw do thing" + let segments = makeSegments( + transcript: transcript, + words: [ + ("hey", 0.0, 0.1), + ("openclaw", 0.2, 0.1), + ("do", 0.9, 0.1), + ("thing", 1.1, 0.1), + ]) + let cmd = VoiceWakeManager.extractCommand( + from: transcript, + segments: segments, + triggers: ["", " ", "openclaw"], + minPostTriggerGap: 0.3) + #expect(cmd == "do thing") + } +} + +private func makeSegments( + transcript: String, + words: [(String, TimeInterval, TimeInterval)]) +-> [WakeWordSegment] { + var searchStart = transcript.startIndex + var output: [WakeWordSegment] = [] + for (word, start, duration) in words { + let range = transcript.range(of: word, range: searchStart../dev/null 2>&1; then + echo "error: swiftformat not found (brew install swiftformat)" >&2 + exit 1 + fi + swiftformat --lint --config "$SRCROOT/../../.swiftformat" \ + --filelist "$SRCROOT/SwiftSources.input.xcfilelist" + - name: SwiftLint + basedOnDependencyAnalysis: false + inputFileLists: + - $(SRCROOT)/SwiftSources.input.xcfilelist + script: | + set -euo pipefail + export PATH="/opt/homebrew/bin:/usr/local/bin:$PATH" + if ! command -v swiftlint >/dev/null 2>&1; then + echo "error: swiftlint not found (brew install swiftlint)" >&2 + exit 1 + fi + swiftlint lint --config "$SRCROOT/.swiftlint.yml" --use-script-input-file-lists + settings: + base: + CODE_SIGN_IDENTITY: "Apple Development" + CODE_SIGN_STYLE: Manual + DEVELOPMENT_TEAM: Y5PE65HELJ + PRODUCT_BUNDLE_IDENTIFIER: ai.openclaw.ios + PROVISIONING_PROFILE_SPECIFIER: "ai.openclaw.ios Development" + SWIFT_VERSION: "6.0" + SWIFT_STRICT_CONCURRENCY: complete + ENABLE_APPINTENTS_METADATA: NO + info: + path: Sources/Info.plist + properties: + CFBundleDisplayName: OpenClaw + CFBundleIconName: AppIcon + CFBundleShortVersionString: "2026.1.27-beta.1" + CFBundleVersion: "20260126" + UILaunchScreen: {} + UIApplicationSceneManifest: + UIApplicationSupportsMultipleScenes: false + UIBackgroundModes: + - audio + NSLocalNetworkUsageDescription: OpenClaw discovers and connects to your OpenClaw gateway on the local network. + NSAppTransportSecurity: + NSAllowsArbitraryLoadsInWebContent: true + NSBonjourServices: + - _openclaw-gw._tcp + NSCameraUsageDescription: OpenClaw can capture photos or short video clips when requested via the gateway. + NSLocationWhenInUseUsageDescription: OpenClaw uses your location when you allow location sharing. + NSLocationAlwaysAndWhenInUseUsageDescription: OpenClaw can share your location in the background when you enable Always. + NSMicrophoneUsageDescription: OpenClaw needs microphone access for voice wake. + NSSpeechRecognitionUsageDescription: OpenClaw uses on-device speech recognition for voice wake. + UISupportedInterfaceOrientations: + - UIInterfaceOrientationPortrait + - UIInterfaceOrientationPortraitUpsideDown + - UIInterfaceOrientationLandscapeLeft + - UIInterfaceOrientationLandscapeRight + UISupportedInterfaceOrientations~ipad: + - UIInterfaceOrientationPortrait + - UIInterfaceOrientationPortraitUpsideDown + - UIInterfaceOrientationLandscapeLeft + - UIInterfaceOrientationLandscapeRight + + OpenClawTests: + type: bundle.unit-test + platform: iOS + sources: + - path: Tests + dependencies: + - target: OpenClaw + - package: Swabble + product: SwabbleKit + - sdk: AppIntents.framework + settings: + base: + PRODUCT_BUNDLE_IDENTIFIER: ai.openclaw.ios.tests + SWIFT_VERSION: "6.0" + SWIFT_STRICT_CONCURRENCY: complete + TEST_HOST: "$(BUILT_PRODUCTS_DIR)/OpenClaw.app/OpenClaw" + BUNDLE_LOADER: "$(TEST_HOST)" + info: + path: Tests/Info.plist + properties: + CFBundleDisplayName: OpenClawTests + CFBundleShortVersionString: "2026.1.27-beta.1" + CFBundleVersion: "20260126" diff --git a/apps/macos/Icon.icon/Assets/openclaw-mac.png b/apps/macos/Icon.icon/Assets/openclaw-mac.png new file mode 100644 index 0000000000000000000000000000000000000000..e79a671cc8710b2e64e65b3518c492db224abc0e --- /dev/null +++ b/apps/macos/Icon.icon/Assets/openclaw-mac.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3da94d4f6d5e6e3d4ec2d92becf960f1d8090ecd67a5674cc6b9794ba3dbb2ef +size 1393705 diff --git a/apps/macos/Icon.icon/icon.json b/apps/macos/Icon.icon/icon.json new file mode 100644 index 0000000000000000000000000000000000000000..6172a47ef23893735fc2553c5101db0905e2451c --- /dev/null +++ b/apps/macos/Icon.icon/icon.json @@ -0,0 +1,36 @@ +{ + "fill" : { + "automatic-gradient" : "extended-srgb:0.00000,0.53333,1.00000,1.00000" + }, + "groups" : [ + { + "layers" : [ + { + "image-name" : "openclaw-mac.png", + "name" : "openclaw-mac", + "position" : { + "scale" : 1.07, + "translation-in-points" : [ + -2, + 0 + ] + } + } + ], + "shadow" : { + "kind" : "neutral", + "opacity" : 0.5 + }, + "translucency" : { + "enabled" : true, + "value" : 0.5 + } + } + ], + "supported-platforms" : { + "circles" : [ + "watchOS" + ], + "squares" : "shared" + } +} diff --git a/apps/macos/Package.resolved b/apps/macos/Package.resolved new file mode 100644 index 0000000000000000000000000000000000000000..0281713738b14a0243f6d7e3cf5ac755f3be6268 --- /dev/null +++ b/apps/macos/Package.resolved @@ -0,0 +1,132 @@ +{ + "originHash" : "1c9c9d251b760ed3234ecff741a88eb4bf42315ad6f50ac7392b187cf226c16c", + "pins" : [ + { + "identity" : "axorcist", + "kind" : "remoteSourceControl", + "location" : "https://github.com/steipete/AXorcist.git", + "state" : { + "revision" : "c75d06f7f93e264a9786edc2b78c04973061cb2f", + "version" : "0.1.0" + } + }, + { + "identity" : "commander", + "kind" : "remoteSourceControl", + "location" : "https://github.com/steipete/Commander.git", + "state" : { + "revision" : "9e349575c8e3c6745e81fe19e5bb5efa01b078ce", + "version" : "0.2.1" + } + }, + { + "identity" : "elevenlabskit", + "kind" : "remoteSourceControl", + "location" : "https://github.com/steipete/ElevenLabsKit", + "state" : { + "revision" : "c8679fbd37416a8780fe43be88a497ff16209e2d", + "version" : "0.1.0" + } + }, + { + "identity" : "menubarextraaccess", + "kind" : "remoteSourceControl", + "location" : "https://github.com/orchetect/MenuBarExtraAccess", + "state" : { + "revision" : "707dff6f55217b3ef5b6be84ced3e83511d4df5c", + "version" : "1.2.2" + } + }, + { + "identity" : "peekaboo", + "kind" : "remoteSourceControl", + "location" : "https://github.com/steipete/Peekaboo.git", + "state" : { + "branch" : "main", + "revision" : "bace59f90bb276f1c6fb613acfda3935ec4a7a90" + } + }, + { + "identity" : "sparkle", + "kind" : "remoteSourceControl", + "location" : "https://github.com/sparkle-project/Sparkle", + "state" : { + "revision" : "5581748cef2bae787496fe6d61139aebe0a451f6", + "version" : "2.8.1" + } + }, + { + "identity" : "swift-algorithms", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-algorithms", + "state" : { + "revision" : "87e50f483c54e6efd60e885f7f5aa946cee68023", + "version" : "1.2.1" + } + }, + { + "identity" : "swift-concurrency-extras", + "kind" : "remoteSourceControl", + "location" : "https://github.com/pointfreeco/swift-concurrency-extras", + "state" : { + "revision" : "5a3825302b1a0d744183200915a47b508c828e6f", + "version" : "1.3.2" + } + }, + { + "identity" : "swift-log", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-log.git", + "state" : { + "revision" : "2778fd4e5a12a8aaa30a3ee8285f4ce54c5f3181", + "version" : "1.9.1" + } + }, + { + "identity" : "swift-numerics", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-numerics.git", + "state" : { + "revision" : "0c0290ff6b24942dadb83a929ffaaa1481df04a2", + "version" : "1.1.1" + } + }, + { + "identity" : "swift-subprocess", + "kind" : "remoteSourceControl", + "location" : "https://github.com/swiftlang/swift-subprocess.git", + "state" : { + "revision" : "ba5888ad7758cbcbe7abebac37860b1652af2d9c", + "version" : "0.3.0" + } + }, + { + "identity" : "swift-system", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-system", + "state" : { + "revision" : "7c6ad0fc39d0763e0b699210e4124afd5041c5df", + "version" : "1.6.4" + } + }, + { + "identity" : "swiftui-math", + "kind" : "remoteSourceControl", + "location" : "https://github.com/gonzalezreal/swiftui-math", + "state" : { + "revision" : "0b5c2cfaaec8d6193db206f675048eeb5ce95f71", + "version" : "0.1.0" + } + }, + { + "identity" : "textual", + "kind" : "remoteSourceControl", + "location" : "https://github.com/gonzalezreal/textual", + "state" : { + "revision" : "5b06b811c0f5313b6b84bbef98c635a630638c38", + "version" : "0.3.1" + } + } + ], + "version" : 3 +} diff --git a/apps/macos/Package.swift b/apps/macos/Package.swift new file mode 100644 index 0000000000000000000000000000000000000000..10ab47b8514069b2121d7de2a881465fe74f0803 --- /dev/null +++ b/apps/macos/Package.swift @@ -0,0 +1,92 @@ +// swift-tools-version: 6.2 +// Package manifest for the OpenClaw macOS companion (menu bar app + IPC library). + +import PackageDescription + +let package = Package( + name: "OpenClaw", + platforms: [ + .macOS(.v15), + ], + products: [ + .library(name: "OpenClawIPC", targets: ["OpenClawIPC"]), + .library(name: "OpenClawDiscovery", targets: ["OpenClawDiscovery"]), + .executable(name: "OpenClaw", targets: ["OpenClaw"]), + .executable(name: "openclaw-mac", targets: ["OpenClawMacCLI"]), + ], + dependencies: [ + .package(url: "https://github.com/orchetect/MenuBarExtraAccess", exact: "1.2.2"), + .package(url: "https://github.com/swiftlang/swift-subprocess.git", from: "0.1.0"), + .package(url: "https://github.com/apple/swift-log.git", from: "1.8.0"), + .package(url: "https://github.com/sparkle-project/Sparkle", from: "2.8.1"), + .package(url: "https://github.com/steipete/Peekaboo.git", branch: "main"), + .package(path: "../shared/OpenClawKit"), + .package(path: "../../Swabble"), + ], + targets: [ + .target( + name: "OpenClawIPC", + dependencies: [], + swiftSettings: [ + .enableUpcomingFeature("StrictConcurrency"), + ]), + .target( + name: "OpenClawDiscovery", + dependencies: [ + .product(name: "OpenClawKit", package: "OpenClawKit"), + ], + path: "Sources/OpenClawDiscovery", + swiftSettings: [ + .enableUpcomingFeature("StrictConcurrency"), + ]), + .executableTarget( + name: "OpenClaw", + dependencies: [ + "OpenClawIPC", + "OpenClawDiscovery", + .product(name: "OpenClawKit", package: "OpenClawKit"), + .product(name: "OpenClawChatUI", package: "OpenClawKit"), + .product(name: "OpenClawProtocol", package: "OpenClawKit"), + .product(name: "SwabbleKit", package: "swabble"), + .product(name: "MenuBarExtraAccess", package: "MenuBarExtraAccess"), + .product(name: "Subprocess", package: "swift-subprocess"), + .product(name: "Logging", package: "swift-log"), + .product(name: "Sparkle", package: "Sparkle"), + .product(name: "PeekabooBridge", package: "Peekaboo"), + .product(name: "PeekabooAutomationKit", package: "Peekaboo"), + ], + exclude: [ + "Resources/Info.plist", + ], + resources: [ + .copy("Resources/OpenClaw.icns"), + .copy("Resources/DeviceModels"), + ], + swiftSettings: [ + .enableUpcomingFeature("StrictConcurrency"), + ]), + .executableTarget( + name: "OpenClawMacCLI", + dependencies: [ + "OpenClawDiscovery", + .product(name: "OpenClawKit", package: "OpenClawKit"), + .product(name: "OpenClawProtocol", package: "OpenClawKit"), + ], + path: "Sources/OpenClawMacCLI", + swiftSettings: [ + .enableUpcomingFeature("StrictConcurrency"), + ]), + .testTarget( + name: "OpenClawIPCTests", + dependencies: [ + "OpenClawIPC", + "OpenClaw", + "OpenClawDiscovery", + .product(name: "OpenClawProtocol", package: "OpenClawKit"), + .product(name: "SwabbleKit", package: "swabble"), + ], + swiftSettings: [ + .enableUpcomingFeature("StrictConcurrency"), + .enableExperimentalFeature("SwiftTesting"), + ]), + ]) diff --git a/apps/macos/README.md b/apps/macos/README.md new file mode 100644 index 0000000000000000000000000000000000000000..05743dc6e2f42a393fcd15e3a981c88de81a7cf1 --- /dev/null +++ b/apps/macos/README.md @@ -0,0 +1,64 @@ +# OpenClaw macOS app (dev + signing) + +## Quick dev run + +```bash +# from repo root +scripts/restart-mac.sh +``` + +Options: + +```bash +scripts/restart-mac.sh --no-sign # fastest dev; ad-hoc signing (TCC permissions do not stick) +scripts/restart-mac.sh --sign # force code signing (requires cert) +``` + +## Packaging flow + +```bash +scripts/package-mac-app.sh +``` + +Creates `dist/OpenClaw.app` and signs it via `scripts/codesign-mac-app.sh`. + +## Signing behavior + +Auto-selects identity (first match): +1) Developer ID Application +2) Apple Distribution +3) Apple Development +4) first available identity + +If none found: +- errors by default +- set `ALLOW_ADHOC_SIGNING=1` or `SIGN_IDENTITY="-"` to ad-hoc sign + +## Team ID audit (Sparkle mismatch guard) + +After signing, we read the app bundle Team ID and compare every Mach-O inside the app. +If any embedded binary has a different Team ID, signing fails. + +Skip the audit: +```bash +SKIP_TEAM_ID_CHECK=1 scripts/package-mac-app.sh +``` + +## Library validation workaround (dev only) + +If Sparkle Team ID mismatch blocks loading (common with Apple Development certs), opt in: + +```bash +DISABLE_LIBRARY_VALIDATION=1 scripts/package-mac-app.sh +``` + +This adds `com.apple.security.cs.disable-library-validation` to app entitlements. +Use for local dev only; keep off for release builds. + +## Useful env flags + +- `SIGN_IDENTITY="Apple Development: Your Name (TEAMID)"` +- `ALLOW_ADHOC_SIGNING=1` (ad-hoc, TCC permissions do not persist) +- `CODESIGN_TIMESTAMP=off` (offline debug) +- `DISABLE_LIBRARY_VALIDATION=1` (dev-only Sparkle workaround) +- `SKIP_TEAM_ID_CHECK=1` (bypass audit) diff --git a/apps/macos/Sources/OpenClaw/AboutSettings.swift b/apps/macos/Sources/OpenClaw/AboutSettings.swift new file mode 100644 index 0000000000000000000000000000000000000000..ede898ebac2e39aee09e52476744395c2115d786 --- /dev/null +++ b/apps/macos/Sources/OpenClaw/AboutSettings.swift @@ -0,0 +1,199 @@ +import SwiftUI + +struct AboutSettings: View { + weak var updater: UpdaterProviding? + @State private var iconHover = false + @AppStorage("autoUpdateEnabled") private var autoCheckEnabled = true + @State private var didLoadUpdaterState = false + + var body: some View { + VStack(spacing: 8) { + let appIcon = NSApplication.shared.applicationIconImage ?? CritterIconRenderer.makeIcon(blink: 0) + Button { + if let url = URL(string: "https://github.com/openclaw/openclaw") { + NSWorkspace.shared.open(url) + } + } label: { + Image(nsImage: appIcon) + .resizable() + .frame(width: 160, height: 160) + .cornerRadius(24) + .shadow(color: self.iconHover ? .accentColor.opacity(0.25) : .clear, radius: 10) + .scaleEffect(self.iconHover ? 1.05 : 1.0) + } + .buttonStyle(.plain) + .focusable(false) + .pointingHandCursor() + .onHover { hover in + withAnimation(.spring(response: 0.3, dampingFraction: 0.72)) { self.iconHover = hover } + } + + VStack(spacing: 3) { + Text("OpenClaw") + .font(.title3.bold()) + Text("Version \(self.versionString)") + .foregroundStyle(.secondary) + if let buildTimestamp { + Text("Built \(buildTimestamp)\(self.buildSuffix)") + .font(.footnote) + .foregroundStyle(.secondary) + } + Text("Menu bar companion for notifications, screenshots, and privileged agent actions.") + .font(.footnote) + .foregroundStyle(.secondary) + .multilineTextAlignment(.center) + .padding(.horizontal, 18) + } + + VStack(alignment: .center, spacing: 6) { + AboutLinkRow( + icon: "chevron.left.slash.chevron.right", + title: "GitHub", + url: "https://github.com/openclaw/openclaw") + AboutLinkRow(icon: "globe", title: "Website", url: "https://openclaw.ai") + AboutLinkRow(icon: "bird", title: "Twitter", url: "https://twitter.com/steipete") + AboutLinkRow(icon: "envelope", title: "Email", url: "mailto:peter@steipete.me") + } + .frame(maxWidth: .infinity) + .multilineTextAlignment(.center) + .padding(.vertical, 10) + + if let updater { + Divider() + .padding(.vertical, 8) + + if updater.isAvailable { + VStack(spacing: 10) { + Toggle("Check for updates automatically", isOn: self.$autoCheckEnabled) + .toggleStyle(.checkbox) + .frame(maxWidth: .infinity, alignment: .center) + + Button("Check for Updates…") { updater.checkForUpdates(nil) } + } + } else { + Text("Updates unavailable in this build.") + .foregroundStyle(.secondary) + .padding(.top, 4) + } + } + + Text("© 2025 Peter Steinberger — MIT License.") + .font(.footnote) + .foregroundStyle(.secondary) + .padding(.top, 4) + + Spacer() + } + .frame(maxWidth: .infinity, maxHeight: .infinity) + .padding(.top, 4) + .padding(.horizontal, 24) + .padding(.bottom, 24) + .onAppear { + guard let updater, !self.didLoadUpdaterState else { return } + // Keep Sparkle’s auto-check setting in sync with the persisted toggle. + updater.automaticallyChecksForUpdates = self.autoCheckEnabled + updater.automaticallyDownloadsUpdates = self.autoCheckEnabled + self.didLoadUpdaterState = true + } + .onChange(of: self.autoCheckEnabled) { _, newValue in + self.updater?.automaticallyChecksForUpdates = newValue + self.updater?.automaticallyDownloadsUpdates = newValue + } + } + + private var versionString: String { + let version = Bundle.main.object(forInfoDictionaryKey: "CFBundleShortVersionString") as? String ?? "dev" + let build = Bundle.main.object(forInfoDictionaryKey: "CFBundleVersion") as? String + return build.map { "\(version) (\($0))" } ?? version + } + + private var buildTimestamp: String? { + guard + let raw = + (Bundle.main.object(forInfoDictionaryKey: "OpenClawBuildTimestamp") as? String) ?? + (Bundle.main.object(forInfoDictionaryKey: "OpenClawBuildTimestamp") as? String) + else { return nil } + let parser = ISO8601DateFormatter() + parser.formatOptions = [.withInternetDateTime] + guard let date = parser.date(from: raw) else { return raw } + + let formatter = DateFormatter() + formatter.dateStyle = .medium + formatter.timeStyle = .short + formatter.locale = .current + return formatter.string(from: date) + } + + private var gitCommit: String { + (Bundle.main.object(forInfoDictionaryKey: "OpenClawGitCommit") as? String) ?? + (Bundle.main.object(forInfoDictionaryKey: "OpenClawGitCommit") as? String) ?? + "unknown" + } + + private var bundleID: String { + Bundle.main.bundleIdentifier ?? "unknown" + } + + private var buildSuffix: String { + let git = self.gitCommit + guard !git.isEmpty, git != "unknown" else { return "" } + + var suffix = " (\(git)" + #if DEBUG + suffix += " DEBUG" + #endif + suffix += ")" + return suffix + } +} + +@MainActor +private struct AboutLinkRow: View { + let icon: String + let title: String + let url: String + + @State private var hovering = false + + var body: some View { + Button { + if let url = URL(string: url) { NSWorkspace.shared.open(url) } + } label: { + HStack(spacing: 6) { + Image(systemName: self.icon) + Text(self.title) + .underline(self.hovering, color: .accentColor) + } + .foregroundColor(.accentColor) + } + .buttonStyle(.plain) + .onHover { self.hovering = $0 } + .pointingHandCursor() + } +} + +private struct AboutMetaRow: View { + let label: String + let value: String + + var body: some View { + HStack { + Text(self.label) + .foregroundStyle(.secondary) + Spacer() + Text(self.value) + .font(.caption.monospaced()) + .foregroundStyle(.primary) + } + } +} + +#if DEBUG +struct AboutSettings_Previews: PreviewProvider { + private static let updater = DisabledUpdaterController() + static var previews: some View { + AboutSettings(updater: updater) + .frame(width: SettingsTab.windowWidth, height: SettingsTab.windowHeight) + } +} +#endif diff --git a/apps/macos/Sources/OpenClaw/AgeFormatting.swift b/apps/macos/Sources/OpenClaw/AgeFormatting.swift new file mode 100644 index 0000000000000000000000000000000000000000..f992c2d95e3c57c0d0b210dfc561452727ac17f2 --- /dev/null +++ b/apps/macos/Sources/OpenClaw/AgeFormatting.swift @@ -0,0 +1,17 @@ +import Foundation + +// Human-friendly age string (e.g., "2m ago"). +func age(from date: Date, now: Date = .init()) -> String { + let seconds = max(0, Int(now.timeIntervalSince(date))) + let minutes = seconds / 60 + let hours = minutes / 60 + let days = hours / 24 + + if seconds < 60 { return "just now" } + if minutes == 1 { return "1 minute ago" } + if minutes < 60 { return "\(minutes)m ago" } + if hours == 1 { return "1 hour ago" } + if hours < 24 { return "\(hours)h ago" } + if days == 1 { return "yesterday" } + return "\(days)d ago" +} diff --git a/apps/macos/Sources/OpenClaw/AgentEventStore.swift b/apps/macos/Sources/OpenClaw/AgentEventStore.swift new file mode 100644 index 0000000000000000000000000000000000000000..780867a32f4d3cca715252d1600bed3125fa1a19 --- /dev/null +++ b/apps/macos/Sources/OpenClaw/AgentEventStore.swift @@ -0,0 +1,22 @@ +import Foundation +import Observation + +@MainActor +@Observable +final class AgentEventStore { + static let shared = AgentEventStore() + + private(set) var events: [ControlAgentEvent] = [] + private let maxEvents = 400 + + func append(_ event: ControlAgentEvent) { + self.events.append(event) + if self.events.count > self.maxEvents { + self.events.removeFirst(self.events.count - self.maxEvents) + } + } + + func clear() { + self.events.removeAll() + } +} diff --git a/apps/macos/Sources/OpenClaw/AgentEventsWindow.swift b/apps/macos/Sources/OpenClaw/AgentEventsWindow.swift new file mode 100644 index 0000000000000000000000000000000000000000..673588cc37923193873d91fe015fa0bae32447cb --- /dev/null +++ b/apps/macos/Sources/OpenClaw/AgentEventsWindow.swift @@ -0,0 +1,109 @@ +import OpenClawProtocol +import SwiftUI + +@MainActor +struct AgentEventsWindow: View { + private let store = AgentEventStore.shared + + var body: some View { + VStack(alignment: .leading, spacing: 6) { + HStack { + Text("Agent Events") + .font(.title3.weight(.semibold)) + Spacer() + Button("Clear") { self.store.clear() } + .buttonStyle(.bordered) + } + .padding(.bottom, 4) + + ScrollView { + LazyVStack(alignment: .leading, spacing: 8) { + ForEach(self.store.events.reversed(), id: \.seq) { evt in + EventRow(event: evt) + } + } + } + } + .padding(12) + .frame(minWidth: 520, minHeight: 360) + } +} + +private struct EventRow: View { + let event: ControlAgentEvent + + var body: some View { + VStack(alignment: .leading, spacing: 2) { + HStack(spacing: 6) { + Text(self.event.stream.uppercased()) + .font(.caption2.weight(.bold)) + .padding(.horizontal, 6) + .padding(.vertical, 2) + .background(self.tint) + .foregroundStyle(Color.white) + .clipShape(RoundedRectangle(cornerRadius: 5, style: .continuous)) + Text("run " + self.event.runId) + .font(.caption.monospaced()) + .foregroundStyle(.secondary) + Spacer() + Text(self.formattedTs) + .font(.caption2) + .foregroundStyle(.secondary) + } + if let json = self.prettyJSON(event.data) { + Text(json) + .font(.caption.monospaced()) + .foregroundStyle(.primary) + .textSelection(.enabled) + .frame(maxWidth: .infinity, alignment: .leading) + .padding(.top, 2) + } + } + .padding(8) + .background( + RoundedRectangle(cornerRadius: 8, style: .continuous) + .fill(Color.primary.opacity(0.04))) + } + + private var tint: Color { + switch self.event.stream { + case "job": .blue + case "tool": .orange + case "assistant": .green + default: .gray + } + } + + private var formattedTs: String { + let date = Date(timeIntervalSince1970: event.ts / 1000) + let f = DateFormatter() + f.dateFormat = "HH:mm:ss.SSS" + return f.string(from: date) + } + + private func prettyJSON(_ dict: [String: OpenClawProtocol.AnyCodable]) -> String? { + let normalized = dict.mapValues { $0.value } + guard JSONSerialization.isValidJSONObject(normalized), + let data = try? JSONSerialization.data(withJSONObject: normalized, options: [.prettyPrinted]), + let str = String(data: data, encoding: .utf8) + else { return nil } + return str + } +} + +struct AgentEventsWindow_Previews: PreviewProvider { + static var previews: some View { + let sample = ControlAgentEvent( + runId: "abc", + seq: 1, + stream: "tool", + ts: Date().timeIntervalSince1970 * 1000, + data: [ + "phase": OpenClawProtocol.AnyCodable("start"), + "name": OpenClawProtocol.AnyCodable("bash"), + ], + summary: nil) + AgentEventStore.shared.append(sample) + return AgentEventsWindow() + } +} diff --git a/apps/macos/Sources/OpenClaw/AgentWorkspace.swift b/apps/macos/Sources/OpenClaw/AgentWorkspace.swift new file mode 100644 index 0000000000000000000000000000000000000000..603f837f45e5ea82e503869af912d016c06d440b --- /dev/null +++ b/apps/macos/Sources/OpenClaw/AgentWorkspace.swift @@ -0,0 +1,340 @@ +import Foundation +import OSLog + +enum AgentWorkspace { + private static let logger = Logger(subsystem: "ai.openclaw", category: "workspace") + static let agentsFilename = "AGENTS.md" + static let soulFilename = "SOUL.md" + static let identityFilename = "IDENTITY.md" + static let userFilename = "USER.md" + static let bootstrapFilename = "BOOTSTRAP.md" + private static let templateDirname = "templates" + private static let ignoredEntries: Set = [".DS_Store", ".git", ".gitignore"] + private static let templateEntries: Set = [ + AgentWorkspace.agentsFilename, + AgentWorkspace.soulFilename, + AgentWorkspace.identityFilename, + AgentWorkspace.userFilename, + AgentWorkspace.bootstrapFilename, + ] + enum BootstrapSafety: Equatable { + case safe + case unsafe(reason: String) + } + + static func displayPath(for url: URL) -> String { + let home = FileManager().homeDirectoryForCurrentUser.path + let path = url.path + if path == home { return "~" } + if path.hasPrefix(home + "/") { + return "~/" + String(path.dropFirst(home.count + 1)) + } + return path + } + + static func resolveWorkspaceURL(from userInput: String?) -> URL { + let trimmed = userInput?.trimmingCharacters(in: .whitespacesAndNewlines) ?? "" + if trimmed.isEmpty { return OpenClawConfigFile.defaultWorkspaceURL() } + let expanded = (trimmed as NSString).expandingTildeInPath + return URL(fileURLWithPath: expanded, isDirectory: true) + } + + static func agentsURL(workspaceURL: URL) -> URL { + workspaceURL.appendingPathComponent(self.agentsFilename) + } + + static func workspaceEntries(workspaceURL: URL) throws -> [String] { + let contents = try FileManager().contentsOfDirectory(atPath: workspaceURL.path) + return contents.filter { !self.ignoredEntries.contains($0) } + } + + static func isWorkspaceEmpty(workspaceURL: URL) -> Bool { + let fm = FileManager() + var isDir: ObjCBool = false + if !fm.fileExists(atPath: workspaceURL.path, isDirectory: &isDir) { + return true + } + guard isDir.boolValue else { return false } + guard let entries = try? self.workspaceEntries(workspaceURL: workspaceURL) else { return false } + return entries.isEmpty + } + + static func isTemplateOnlyWorkspace(workspaceURL: URL) -> Bool { + guard let entries = try? self.workspaceEntries(workspaceURL: workspaceURL) else { return false } + guard !entries.isEmpty else { return true } + return Set(entries).isSubset(of: self.templateEntries) + } + + static func bootstrapSafety(for workspaceURL: URL) -> BootstrapSafety { + let fm = FileManager() + var isDir: ObjCBool = false + if !fm.fileExists(atPath: workspaceURL.path, isDirectory: &isDir) { + return .safe + } + if !isDir.boolValue { + return .unsafe(reason: "Workspace path points to a file.") + } + let agentsURL = self.agentsURL(workspaceURL: workspaceURL) + if fm.fileExists(atPath: agentsURL.path) { + return .safe + } + do { + let entries = try self.workspaceEntries(workspaceURL: workspaceURL) + return entries.isEmpty + ? .safe + : .unsafe(reason: "Folder isn't empty. Choose a new folder or add AGENTS.md first.") + } catch { + return .unsafe(reason: "Couldn't inspect the workspace folder.") + } + } + + static func bootstrap(workspaceURL: URL) throws -> URL { + let shouldSeedBootstrap = self.isWorkspaceEmpty(workspaceURL: workspaceURL) + try FileManager().createDirectory(at: workspaceURL, withIntermediateDirectories: true) + let agentsURL = self.agentsURL(workspaceURL: workspaceURL) + if !FileManager().fileExists(atPath: agentsURL.path) { + try self.defaultTemplate().write(to: agentsURL, atomically: true, encoding: .utf8) + self.logger.info("Created AGENTS.md at \(agentsURL.path, privacy: .public)") + } + let soulURL = workspaceURL.appendingPathComponent(self.soulFilename) + if !FileManager().fileExists(atPath: soulURL.path) { + try self.defaultSoulTemplate().write(to: soulURL, atomically: true, encoding: .utf8) + self.logger.info("Created SOUL.md at \(soulURL.path, privacy: .public)") + } + let identityURL = workspaceURL.appendingPathComponent(self.identityFilename) + if !FileManager().fileExists(atPath: identityURL.path) { + try self.defaultIdentityTemplate().write(to: identityURL, atomically: true, encoding: .utf8) + self.logger.info("Created IDENTITY.md at \(identityURL.path, privacy: .public)") + } + let userURL = workspaceURL.appendingPathComponent(self.userFilename) + if !FileManager().fileExists(atPath: userURL.path) { + try self.defaultUserTemplate().write(to: userURL, atomically: true, encoding: .utf8) + self.logger.info("Created USER.md at \(userURL.path, privacy: .public)") + } + let bootstrapURL = workspaceURL.appendingPathComponent(self.bootstrapFilename) + if shouldSeedBootstrap, !FileManager().fileExists(atPath: bootstrapURL.path) { + try self.defaultBootstrapTemplate().write(to: bootstrapURL, atomically: true, encoding: .utf8) + self.logger.info("Created BOOTSTRAP.md at \(bootstrapURL.path, privacy: .public)") + } + return agentsURL + } + + static func needsBootstrap(workspaceURL: URL) -> Bool { + let fm = FileManager() + var isDir: ObjCBool = false + if !fm.fileExists(atPath: workspaceURL.path, isDirectory: &isDir) { + return true + } + guard isDir.boolValue else { return true } + if self.hasIdentity(workspaceURL: workspaceURL) { + return false + } + let bootstrapURL = workspaceURL.appendingPathComponent(self.bootstrapFilename) + guard fm.fileExists(atPath: bootstrapURL.path) else { return false } + return self.isTemplateOnlyWorkspace(workspaceURL: workspaceURL) + } + + static func hasIdentity(workspaceURL: URL) -> Bool { + let identityURL = workspaceURL.appendingPathComponent(self.identityFilename) + guard let contents = try? String(contentsOf: identityURL, encoding: .utf8) else { return false } + return self.identityLinesHaveValues(contents) + } + + private static func identityLinesHaveValues(_ content: String) -> Bool { + for line in content.split(separator: "\n") { + let trimmed = line.trimmingCharacters(in: .whitespacesAndNewlines) + guard trimmed.hasPrefix("-"), let colon = trimmed.firstIndex(of: ":") else { continue } + let value = trimmed[trimmed.index(after: colon)...].trimmingCharacters(in: .whitespacesAndNewlines) + if !value.isEmpty { + return true + } + } + return false + } + + static func defaultTemplate() -> String { + let fallback = """ + # AGENTS.md - OpenClaw Workspace + + This folder is the assistant's working directory. + + ## First run (one-time) + - If BOOTSTRAP.md exists, follow its ritual and delete it once complete. + - Your agent identity lives in IDENTITY.md. + - Your profile lives in USER.md. + + ## Backup tip (recommended) + If you treat this workspace as the agent's "memory", make it a git repo (ideally private) so identity + and notes are backed up. + + ```bash + git init + git add AGENTS.md + git commit -m "Add agent workspace" + ``` + + ## Safety defaults + - Don't exfiltrate secrets or private data. + - Don't run destructive commands unless explicitly asked. + - Be concise in chat; write longer output to files in this workspace. + + ## Daily memory (recommended) + - Keep a short daily log at memory/YYYY-MM-DD.md (create memory/ if needed). + - On session start, read today + yesterday if present. + - Capture durable facts, preferences, and decisions; avoid secrets. + + ## Customize + - Add your preferred style, rules, and "memory" here. + """ + return self.loadTemplate(named: self.agentsFilename, fallback: fallback) + } + + static func defaultSoulTemplate() -> String { + let fallback = """ + # SOUL.md - Persona & Boundaries + + Describe who the assistant is, tone, and boundaries. + + - Keep replies concise and direct. + - Ask clarifying questions when needed. + - Never send streaming/partial replies to external messaging surfaces. + """ + return self.loadTemplate(named: self.soulFilename, fallback: fallback) + } + + static func defaultIdentityTemplate() -> String { + let fallback = """ + # IDENTITY.md - Agent Identity + + - Name: + - Creature: + - Vibe: + - Emoji: + """ + return self.loadTemplate(named: self.identityFilename, fallback: fallback) + } + + static func defaultUserTemplate() -> String { + let fallback = """ + # USER.md - User Profile + + - Name: + - Preferred address: + - Pronouns (optional): + - Timezone (optional): + - Notes: + """ + return self.loadTemplate(named: self.userFilename, fallback: fallback) + } + + static func defaultBootstrapTemplate() -> String { + let fallback = """ + # BOOTSTRAP.md - First Run Ritual (delete after) + + Hello. I was just born. + + ## Your mission + Start a short, playful conversation and learn: + - Who am I? + - What am I? + - Who are you? + - How should I call you? + + ## How to ask (cute + helpful) + Say: + "Hello! I was just born. Who am I? What am I? Who are you? How should I call you?" + + Then offer suggestions: + - 3-5 name ideas. + - 3-5 creature/vibe combos. + - 5 emoji ideas. + + ## Write these files + After the user chooses, update: + + 1) IDENTITY.md + - Name + - Creature + - Vibe + - Emoji + + 2) USER.md + - Name + - Preferred address + - Pronouns (optional) + - Timezone (optional) + - Notes + + 3) ~/.openclaw/openclaw.json + Set identity.name, identity.theme, identity.emoji to match IDENTITY.md. + + ## Cleanup + Delete BOOTSTRAP.md once this is complete. + """ + return self.loadTemplate(named: self.bootstrapFilename, fallback: fallback) + } + + private static func loadTemplate(named: String, fallback: String) -> String { + for url in self.templateURLs(named: named) { + if let content = try? String(contentsOf: url, encoding: .utf8) { + let stripped = self.stripFrontMatter(content) + if !stripped.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty { + return stripped + } + } + } + return fallback + } + + private static func templateURLs(named: String) -> [URL] { + var urls: [URL] = [] + if let resource = Bundle.main.url( + forResource: named.replacingOccurrences(of: ".md", with: ""), + withExtension: "md", + subdirectory: self.templateDirname) + { + urls.append(resource) + } + if let resource = Bundle.main.url( + forResource: named, + withExtension: nil, + subdirectory: self.templateDirname) + { + urls.append(resource) + } + if let dev = self.devTemplateURL(named: named) { + urls.append(dev) + } + let cwd = URL(fileURLWithPath: FileManager().currentDirectoryPath) + urls.append(cwd.appendingPathComponent("docs") + .appendingPathComponent(self.templateDirname) + .appendingPathComponent(named)) + return urls + } + + private static func devTemplateURL(named: String) -> URL? { + let sourceURL = URL(fileURLWithPath: #filePath) + let repoRoot = sourceURL + .deletingLastPathComponent() + .deletingLastPathComponent() + .deletingLastPathComponent() + .deletingLastPathComponent() + .deletingLastPathComponent() + return repoRoot.appendingPathComponent("docs") + .appendingPathComponent(self.templateDirname) + .appendingPathComponent(named) + } + + private static func stripFrontMatter(_ content: String) -> String { + guard content.hasPrefix("---") else { return content } + let start = content.index(content.startIndex, offsetBy: 3) + guard let range = content.range(of: "\n---", range: start.. = { + if ProcessInfo.processInfo.isRunningTests { + return Empty(completeImmediately: false).eraseToAnyPublisher() + } + return Timer.publish(every: 0.4, on: .main, in: .common) + .autoconnect() + .eraseToAnyPublisher() + }() + + var body: some View { + VStack(alignment: .leading, spacing: 10) { + if self.connectionMode != .local { + Text("Gateway isn’t running locally; OAuth must be created on the gateway host.") + .font(.footnote) + .foregroundStyle(.secondary) + .fixedSize(horizontal: false, vertical: true) + } + + HStack(spacing: 10) { + Circle() + .fill(self.oauthStatus.isConnected ? Color.green : Color.orange) + .frame(width: 8, height: 8) + Text(self.oauthStatus.shortDescription) + .font(.footnote.weight(.semibold)) + .foregroundStyle(.secondary) + Spacer() + Button("Reveal") { + NSWorkspace.shared.activateFileViewerSelecting([OpenClawOAuthStore.oauthURL()]) + } + .buttonStyle(.bordered) + .disabled(!FileManager().fileExists(atPath: OpenClawOAuthStore.oauthURL().path)) + + Button("Refresh") { + self.refresh() + } + .buttonStyle(.bordered) + } + + Text(OpenClawOAuthStore.oauthURL().path) + .font(.caption.monospaced()) + .foregroundStyle(.secondary) + .lineLimit(1) + .truncationMode(.middle) + .textSelection(.enabled) + + HStack(spacing: 12) { + Button { + self.startOAuth() + } label: { + if self.busy { + ProgressView().controlSize(.small) + } else { + Text(self.oauthStatus.isConnected ? "Re-auth (OAuth)" : "Open sign-in (OAuth)") + } + } + .buttonStyle(.borderedProminent) + .disabled(self.connectionMode != .local || self.busy) + + if self.pkce != nil { + Button("Cancel") { + self.pkce = nil + self.code = "" + self.statusText = nil + } + .buttonStyle(.bordered) + .disabled(self.busy) + } + } + + if self.pkce != nil { + VStack(alignment: .leading, spacing: 8) { + Text("Paste `code#state`") + .font(.footnote.weight(.semibold)) + .foregroundStyle(.secondary) + + TextField("code#state", text: self.$code) + .textFieldStyle(.roundedBorder) + .disabled(self.busy) + + Toggle("Auto-detect from clipboard", isOn: self.$autoDetectClipboard) + .font(.footnote) + .foregroundStyle(.secondary) + .disabled(self.busy) + + Toggle("Auto-connect when detected", isOn: self.$autoConnectClipboard) + .font(.footnote) + .foregroundStyle(.secondary) + .disabled(self.busy) + + Button("Connect") { + Task { await self.finishOAuth() } + } + .buttonStyle(.bordered) + .disabled(self.busy || self.connectionMode != .local || self.code + .trimmingCharacters(in: .whitespacesAndNewlines) + .isEmpty) + } + } + + if let statusText, !statusText.isEmpty { + Text(statusText) + .font(.footnote) + .foregroundStyle(.secondary) + .fixedSize(horizontal: false, vertical: true) + } + } + .onAppear { + self.refresh() + } + .onReceive(Self.clipboardPoll) { _ in + self.pollClipboardIfNeeded() + } + } + + private func refresh() { + let imported = OpenClawOAuthStore.importLegacyAnthropicOAuthIfNeeded() + self.oauthStatus = OpenClawOAuthStore.anthropicOAuthStatus() + if imported != nil { + self.statusText = "Imported existing OAuth credentials." + } + } + + private func startOAuth() { + guard self.connectionMode == .local else { return } + guard !self.busy else { return } + self.busy = true + defer { self.busy = false } + + do { + let pkce = try AnthropicOAuth.generatePKCE() + self.pkce = pkce + let url = AnthropicOAuth.buildAuthorizeURL(pkce: pkce) + NSWorkspace.shared.open(url) + self.statusText = "Browser opened. After approving, paste the `code#state` value here." + } catch { + self.statusText = "Failed to start OAuth: \(error.localizedDescription)" + } + } + + @MainActor + private func finishOAuth() async { + guard self.connectionMode == .local else { return } + guard !self.busy else { return } + guard let pkce = self.pkce else { return } + self.busy = true + defer { self.busy = false } + + guard let parsed = AnthropicOAuthCodeState.parse(from: self.code) else { + self.statusText = "OAuth failed: missing or invalid code/state." + return + } + + do { + let creds = try await AnthropicOAuth.exchangeCode( + code: parsed.code, + state: parsed.state, + verifier: pkce.verifier) + try OpenClawOAuthStore.saveAnthropicOAuth(creds) + self.refresh() + self.pkce = nil + self.code = "" + self.statusText = "Connected. OpenClaw can now use Claude via OAuth." + } catch { + self.statusText = "OAuth failed: \(error.localizedDescription)" + } + } + + private func pollClipboardIfNeeded() { + guard self.connectionMode == .local else { return } + guard self.pkce != nil else { return } + guard !self.busy else { return } + guard self.autoDetectClipboard else { return } + + let pb = NSPasteboard.general + let changeCount = pb.changeCount + guard changeCount != self.lastPasteboardChangeCount else { return } + self.lastPasteboardChangeCount = changeCount + + guard let raw = pb.string(forType: .string), !raw.isEmpty else { return } + guard let parsed = AnthropicOAuthCodeState.parse(from: raw) else { return } + guard let pkce = self.pkce, parsed.state == pkce.verifier else { return } + + let next = "\(parsed.code)#\(parsed.state)" + if self.code != next { + self.code = next + self.statusText = "Detected `code#state` from clipboard." + } + + guard self.autoConnectClipboard else { return } + Task { await self.finishOAuth() } + } +} + +#if DEBUG +extension AnthropicAuthControls { + init( + connectionMode: AppState.ConnectionMode, + oauthStatus: OpenClawOAuthStore.AnthropicOAuthStatus, + pkce: AnthropicOAuth.PKCE? = nil, + code: String = "", + busy: Bool = false, + statusText: String? = nil, + autoDetectClipboard: Bool = true, + autoConnectClipboard: Bool = true) + { + self.connectionMode = connectionMode + self._oauthStatus = State(initialValue: oauthStatus) + self._pkce = State(initialValue: pkce) + self._code = State(initialValue: code) + self._busy = State(initialValue: busy) + self._statusText = State(initialValue: statusText) + self._autoDetectClipboard = State(initialValue: autoDetectClipboard) + self._autoConnectClipboard = State(initialValue: autoConnectClipboard) + self._lastPasteboardChangeCount = State(initialValue: NSPasteboard.general.changeCount) + } +} +#endif diff --git a/apps/macos/Sources/OpenClaw/AnthropicOAuth.swift b/apps/macos/Sources/OpenClaw/AnthropicOAuth.swift new file mode 100644 index 0000000000000000000000000000000000000000..408b881ba8fc684fa0d4ccedf5c47434a0aadf43 --- /dev/null +++ b/apps/macos/Sources/OpenClaw/AnthropicOAuth.swift @@ -0,0 +1,384 @@ +import CryptoKit +import Foundation +import OSLog +import Security + +struct AnthropicOAuthCredentials: Codable { + let type: String + let refresh: String + let access: String + let expires: Int64 +} + +enum AnthropicAuthMode: Equatable { + case oauthFile + case oauthEnv + case apiKeyEnv + case missing + + var shortLabel: String { + switch self { + case .oauthFile: "OAuth (OpenClaw token file)" + case .oauthEnv: "OAuth (env var)" + case .apiKeyEnv: "API key (env var)" + case .missing: "Missing credentials" + } + } + + var isConfigured: Bool { + switch self { + case .missing: false + case .oauthFile, .oauthEnv, .apiKeyEnv: true + } + } +} + +enum AnthropicAuthResolver { + static func resolve( + environment: [String: String] = ProcessInfo.processInfo.environment, + oauthStatus: OpenClawOAuthStore.AnthropicOAuthStatus = OpenClawOAuthStore + .anthropicOAuthStatus()) -> AnthropicAuthMode + { + if oauthStatus.isConnected { return .oauthFile } + + if let token = environment["ANTHROPIC_OAUTH_TOKEN"]?.trimmingCharacters(in: .whitespacesAndNewlines), + !token.isEmpty + { + return .oauthEnv + } + + if let key = environment["ANTHROPIC_API_KEY"]?.trimmingCharacters(in: .whitespacesAndNewlines), + !key.isEmpty + { + return .apiKeyEnv + } + + return .missing + } +} + +enum AnthropicOAuth { + private static let logger = Logger(subsystem: "ai.openclaw", category: "anthropic-oauth") + + private static let clientId = "9d1c250a-e61b-44d9-88ed-5944d1962f5e" + private static let authorizeURL = URL(string: "https://claude.ai/oauth/authorize")! + private static let tokenURL = URL(string: "https://console.anthropic.com/v1/oauth/token")! + private static let redirectURI = "https://console.anthropic.com/oauth/code/callback" + private static let scopes = "org:create_api_key user:profile user:inference" + + struct PKCE { + let verifier: String + let challenge: String + } + + static func generatePKCE() throws -> PKCE { + var bytes = [UInt8](repeating: 0, count: 32) + let status = SecRandomCopyBytes(kSecRandomDefault, bytes.count, &bytes) + guard status == errSecSuccess else { + throw NSError(domain: NSOSStatusErrorDomain, code: Int(status)) + } + let verifier = Data(bytes).base64URLEncodedString() + let hash = SHA256.hash(data: Data(verifier.utf8)) + let challenge = Data(hash).base64URLEncodedString() + return PKCE(verifier: verifier, challenge: challenge) + } + + static func buildAuthorizeURL(pkce: PKCE) -> URL { + var components = URLComponents(url: self.authorizeURL, resolvingAgainstBaseURL: false)! + components.queryItems = [ + URLQueryItem(name: "code", value: "true"), + URLQueryItem(name: "client_id", value: self.clientId), + URLQueryItem(name: "response_type", value: "code"), + URLQueryItem(name: "redirect_uri", value: self.redirectURI), + URLQueryItem(name: "scope", value: self.scopes), + URLQueryItem(name: "code_challenge", value: pkce.challenge), + URLQueryItem(name: "code_challenge_method", value: "S256"), + // Match legacy flow: state is the verifier. + URLQueryItem(name: "state", value: pkce.verifier), + ] + return components.url! + } + + static func exchangeCode( + code: String, + state: String, + verifier: String) async throws -> AnthropicOAuthCredentials + { + let payload: [String: Any] = [ + "grant_type": "authorization_code", + "client_id": self.clientId, + "code": code, + "state": state, + "redirect_uri": self.redirectURI, + "code_verifier": verifier, + ] + let body = try JSONSerialization.data(withJSONObject: payload, options: []) + + var request = URLRequest(url: self.tokenURL) + request.httpMethod = "POST" + request.httpBody = body + request.setValue("application/json", forHTTPHeaderField: "Content-Type") + + let (data, response) = try await URLSession.shared.data(for: request) + guard let http = response as? HTTPURLResponse else { + throw URLError(.badServerResponse) + } + guard (200..<300).contains(http.statusCode) else { + let text = String(data: data, encoding: .utf8) ?? "" + throw NSError( + domain: "AnthropicOAuth", + code: http.statusCode, + userInfo: [NSLocalizedDescriptionKey: "Token exchange failed: \(text)"]) + } + + let decoded = try JSONSerialization.jsonObject(with: data) as? [String: Any] + let access = decoded?["access_token"] as? String + let refresh = decoded?["refresh_token"] as? String + let expiresIn = decoded?["expires_in"] as? Double + guard let access, let refresh, let expiresIn else { + throw NSError(domain: "AnthropicOAuth", code: 0, userInfo: [ + NSLocalizedDescriptionKey: "Unexpected token response.", + ]) + } + + // Match legacy flow: expiresAt = now + expires_in - 5 minutes. + let expiresAtMs = Int64(Date().timeIntervalSince1970 * 1000) + + Int64(expiresIn * 1000) + - Int64(5 * 60 * 1000) + + self.logger.info("Anthropic OAuth exchange ok; expiresAtMs=\(expiresAtMs, privacy: .public)") + return AnthropicOAuthCredentials(type: "oauth", refresh: refresh, access: access, expires: expiresAtMs) + } + + static func refresh(refreshToken: String) async throws -> AnthropicOAuthCredentials { + let payload: [String: Any] = [ + "grant_type": "refresh_token", + "client_id": self.clientId, + "refresh_token": refreshToken, + ] + let body = try JSONSerialization.data(withJSONObject: payload, options: []) + + var request = URLRequest(url: self.tokenURL) + request.httpMethod = "POST" + request.httpBody = body + request.setValue("application/json", forHTTPHeaderField: "Content-Type") + + let (data, response) = try await URLSession.shared.data(for: request) + guard let http = response as? HTTPURLResponse else { + throw URLError(.badServerResponse) + } + guard (200..<300).contains(http.statusCode) else { + let text = String(data: data, encoding: .utf8) ?? "" + throw NSError( + domain: "AnthropicOAuth", + code: http.statusCode, + userInfo: [NSLocalizedDescriptionKey: "Token refresh failed: \(text)"]) + } + + let decoded = try JSONSerialization.jsonObject(with: data) as? [String: Any] + let access = decoded?["access_token"] as? String + let refresh = (decoded?["refresh_token"] as? String) ?? refreshToken + let expiresIn = decoded?["expires_in"] as? Double + guard let access, let expiresIn else { + throw NSError(domain: "AnthropicOAuth", code: 0, userInfo: [ + NSLocalizedDescriptionKey: "Unexpected token response.", + ]) + } + + let expiresAtMs = Int64(Date().timeIntervalSince1970 * 1000) + + Int64(expiresIn * 1000) + - Int64(5 * 60 * 1000) + + self.logger.info("Anthropic OAuth refresh ok; expiresAtMs=\(expiresAtMs, privacy: .public)") + return AnthropicOAuthCredentials(type: "oauth", refresh: refresh, access: access, expires: expiresAtMs) + } +} + +enum OpenClawOAuthStore { + static let oauthFilename = "oauth.json" + private static let providerKey = "anthropic" + private static let openclawOAuthDirEnv = "OPENCLAW_OAUTH_DIR" + private static let legacyPiDirEnv = "PI_CODING_AGENT_DIR" + + enum AnthropicOAuthStatus: Equatable { + case missingFile + case unreadableFile + case invalidJSON + case missingProviderEntry + case missingTokens + case connected(expiresAtMs: Int64?) + + var isConnected: Bool { + if case .connected = self { return true } + return false + } + + var shortDescription: String { + switch self { + case .missingFile: "OpenClaw OAuth token file not found" + case .unreadableFile: "OpenClaw OAuth token file not readable" + case .invalidJSON: "OpenClaw OAuth token file invalid" + case .missingProviderEntry: "No Anthropic entry in OpenClaw OAuth token file" + case .missingTokens: "Anthropic entry missing tokens" + case .connected: "OpenClaw OAuth credentials found" + } + } + } + + static func oauthDir() -> URL { + if let override = ProcessInfo.processInfo.environment[self.openclawOAuthDirEnv]? + .trimmingCharacters(in: .whitespacesAndNewlines), + !override.isEmpty + { + let expanded = NSString(string: override).expandingTildeInPath + return URL(fileURLWithPath: expanded, isDirectory: true) + } + let home = FileManager().homeDirectoryForCurrentUser + let preferred = home.appendingPathComponent(".openclaw", isDirectory: true) + .appendingPathComponent("credentials", isDirectory: true) + return preferred + } + + static func oauthURL() -> URL { + self.oauthDir().appendingPathComponent(self.oauthFilename) + } + + static func legacyOAuthURLs() -> [URL] { + var urls: [URL] = [] + let env = ProcessInfo.processInfo.environment + if let override = env[self.legacyPiDirEnv]?.trimmingCharacters(in: .whitespacesAndNewlines), + !override.isEmpty + { + let expanded = NSString(string: override).expandingTildeInPath + urls.append(URL(fileURLWithPath: expanded, isDirectory: true).appendingPathComponent(self.oauthFilename)) + } + + let home = FileManager().homeDirectoryForCurrentUser + urls.append(home.appendingPathComponent(".pi/agent/\(self.oauthFilename)")) + urls.append(home.appendingPathComponent(".claude/\(self.oauthFilename)")) + urls.append(home.appendingPathComponent(".config/claude/\(self.oauthFilename)")) + urls.append(home.appendingPathComponent(".config/anthropic/\(self.oauthFilename)")) + + var seen = Set() + return urls.filter { url in + let path = url.standardizedFileURL.path + if seen.contains(path) { return false } + seen.insert(path) + return true + } + } + + static func importLegacyAnthropicOAuthIfNeeded() -> URL? { + let dest = self.oauthURL() + guard !FileManager().fileExists(atPath: dest.path) else { return nil } + + for url in self.legacyOAuthURLs() { + guard FileManager().fileExists(atPath: url.path) else { continue } + guard self.anthropicOAuthStatus(at: url).isConnected else { continue } + guard let storage = self.loadStorage(at: url) else { continue } + do { + try self.saveStorage(storage) + return url + } catch { + continue + } + } + + return nil + } + + static func anthropicOAuthStatus() -> AnthropicOAuthStatus { + self.anthropicOAuthStatus(at: self.oauthURL()) + } + + static func hasAnthropicOAuth() -> Bool { + self.anthropicOAuthStatus().isConnected + } + + static func anthropicOAuthStatus(at url: URL) -> AnthropicOAuthStatus { + guard FileManager().fileExists(atPath: url.path) else { return .missingFile } + + guard let data = try? Data(contentsOf: url) else { return .unreadableFile } + guard let json = try? JSONSerialization.jsonObject(with: data, options: []) else { return .invalidJSON } + guard let storage = json as? [String: Any] else { return .invalidJSON } + guard let rawEntry = storage[self.providerKey] else { return .missingProviderEntry } + guard let entry = rawEntry as? [String: Any] else { return .invalidJSON } + + let refresh = self.firstString(in: entry, keys: ["refresh", "refresh_token", "refreshToken"]) + let access = self.firstString(in: entry, keys: ["access", "access_token", "accessToken"]) + guard refresh?.isEmpty == false, access?.isEmpty == false else { return .missingTokens } + + let expiresAny = entry["expires"] ?? entry["expires_at"] ?? entry["expiresAt"] + let expiresAtMs: Int64? = if let ms = expiresAny as? Int64 { + ms + } else if let number = expiresAny as? NSNumber { + number.int64Value + } else if let ms = expiresAny as? Double { + Int64(ms) + } else { + nil + } + + return .connected(expiresAtMs: expiresAtMs) + } + + static func loadAnthropicOAuthRefreshToken() -> String? { + let url = self.oauthURL() + guard let storage = self.loadStorage(at: url) else { return nil } + guard let rawEntry = storage[self.providerKey] as? [String: Any] else { return nil } + let refresh = self.firstString(in: rawEntry, keys: ["refresh", "refresh_token", "refreshToken"]) + return refresh?.trimmingCharacters(in: .whitespacesAndNewlines) + } + + private static func firstString(in dict: [String: Any], keys: [String]) -> String? { + for key in keys { + if let value = dict[key] as? String { return value } + } + return nil + } + + private static func loadStorage(at url: URL) -> [String: Any]? { + guard let data = try? Data(contentsOf: url) else { return nil } + guard let json = try? JSONSerialization.jsonObject(with: data, options: []) else { return nil } + return json as? [String: Any] + } + + static func saveAnthropicOAuth(_ creds: AnthropicOAuthCredentials) throws { + let url = self.oauthURL() + let existing: [String: Any] = self.loadStorage(at: url) ?? [:] + + var updated = existing + updated[self.providerKey] = [ + "type": creds.type, + "refresh": creds.refresh, + "access": creds.access, + "expires": creds.expires, + ] + + try self.saveStorage(updated) + } + + private static func saveStorage(_ storage: [String: Any]) throws { + let dir = self.oauthDir() + try FileManager().createDirectory( + at: dir, + withIntermediateDirectories: true, + attributes: [.posixPermissions: 0o700]) + + let url = self.oauthURL() + let data = try JSONSerialization.data( + withJSONObject: storage, + options: [.prettyPrinted, .sortedKeys]) + try data.write(to: url, options: [.atomic]) + try FileManager().setAttributes([.posixPermissions: 0o600], ofItemAtPath: url.path) + } +} + +extension Data { + fileprivate func base64URLEncodedString() -> String { + self.base64EncodedString() + .replacingOccurrences(of: "+", with: "-") + .replacingOccurrences(of: "/", with: "_") + .replacingOccurrences(of: "=", with: "") + } +} diff --git a/apps/macos/Sources/OpenClaw/AnthropicOAuthCodeState.swift b/apps/macos/Sources/OpenClaw/AnthropicOAuthCodeState.swift new file mode 100644 index 0000000000000000000000000000000000000000..2a88898c34df2a03a2e8334a1c2a0d8145ed2377 --- /dev/null +++ b/apps/macos/Sources/OpenClaw/AnthropicOAuthCodeState.swift @@ -0,0 +1,59 @@ +import Foundation + +enum AnthropicOAuthCodeState { + struct Parsed: Equatable { + let code: String + let state: String + } + + /// Extracts a `code#state` payload from arbitrary text. + /// + /// Supports: + /// - raw `code#state` + /// - OAuth callback URLs containing `code=` and `state=` query params + /// - surrounding text/backticks from instructions pages + static func extract(from raw: String) -> String? { + let text = raw.trimmingCharacters(in: .whitespacesAndNewlines) + .trimmingCharacters(in: CharacterSet(charactersIn: "`")) + if text.isEmpty { return nil } + + if let fromURL = self.extractFromURL(text) { return fromURL } + if let fromToken = self.extractFromToken(text) { return fromToken } + return nil + } + + static func parse(from raw: String) -> Parsed? { + guard let extracted = self.extract(from: raw) else { return nil } + let parts = extracted.split(separator: "#", maxSplits: 1).map(String.init) + let code = parts.first ?? "" + let state = parts.count > 1 ? parts[1] : "" + guard !code.isEmpty, !state.isEmpty else { return nil } + return Parsed(code: code, state: state) + } + + private static func extractFromURL(_ text: String) -> String? { + // Users might copy the callback URL from the browser address bar. + guard let components = URLComponents(string: text), + let items = components.queryItems, + let code = items.first(where: { $0.name == "code" })?.value, + let state = items.first(where: { $0.name == "state" })?.value, + !code.isEmpty, !state.isEmpty + else { return nil } + + return "\(code)#\(state)" + } + + private static func extractFromToken(_ text: String) -> String? { + // Base64url-ish tokens; keep this fairly strict to avoid false positives. + let pattern = #"([A-Za-z0-9._~-]{8,})#([A-Za-z0-9._~-]{8,})"# + guard let re = try? NSRegularExpression(pattern: pattern) else { return nil } + + let range = NSRange(text.startIndex..? + + private func ifNotPreview(_ action: () -> Void) { + guard !self.isPreview else { return } + action() + } + + enum ConnectionMode: String { + case unconfigured + case local + case remote + } + + enum RemoteTransport: String { + case ssh + case direct + } + + var isPaused: Bool { + didSet { self.ifNotPreview { UserDefaults.standard.set(self.isPaused, forKey: pauseDefaultsKey) } } + } + + var launchAtLogin: Bool { + didSet { + guard !self.isInitializing else { return } + self.ifNotPreview { Task { AppStateStore.updateLaunchAtLogin(enabled: self.launchAtLogin) } } + } + } + + var onboardingSeen: Bool { + didSet { self.ifNotPreview { UserDefaults.standard.set(self.onboardingSeen, forKey: onboardingSeenKey) } + } + } + + var debugPaneEnabled: Bool { + didSet { + self.ifNotPreview { UserDefaults.standard.set(self.debugPaneEnabled, forKey: debugPaneEnabledKey) } + CanvasManager.shared.refreshDebugStatus() + } + } + + var swabbleEnabled: Bool { + didSet { + self.ifNotPreview { + UserDefaults.standard.set(self.swabbleEnabled, forKey: swabbleEnabledKey) + Task { await VoiceWakeRuntime.shared.refresh(state: self) } + } + } + } + + var swabbleTriggerWords: [String] { + didSet { + // Preserve the raw editing state; sanitization happens when we actually use the triggers. + self.ifNotPreview { + UserDefaults.standard.set(self.swabbleTriggerWords, forKey: swabbleTriggersKey) + if self.swabbleEnabled { + Task { await VoiceWakeRuntime.shared.refresh(state: self) } + } + self.scheduleVoiceWakeGlobalSyncIfNeeded() + } + } + } + + var voiceWakeTriggerChime: VoiceWakeChime { + didSet { self.ifNotPreview { self.storeChime(self.voiceWakeTriggerChime, key: voiceWakeTriggerChimeKey) } } + } + + var voiceWakeSendChime: VoiceWakeChime { + didSet { self.ifNotPreview { self.storeChime(self.voiceWakeSendChime, key: voiceWakeSendChimeKey) } } + } + + var iconAnimationsEnabled: Bool { + didSet { self.ifNotPreview { UserDefaults.standard.set( + self.iconAnimationsEnabled, + forKey: iconAnimationsEnabledKey) } } + } + + var showDockIcon: Bool { + didSet { + self.ifNotPreview { + UserDefaults.standard.set(self.showDockIcon, forKey: showDockIconKey) + AppActivationPolicy.apply(showDockIcon: self.showDockIcon) + } + } + } + + var voiceWakeMicID: String { + didSet { + self.ifNotPreview { + UserDefaults.standard.set(self.voiceWakeMicID, forKey: voiceWakeMicKey) + if self.swabbleEnabled { + Task { await VoiceWakeRuntime.shared.refresh(state: self) } + } + } + } + } + + var voiceWakeMicName: String { + didSet { self.ifNotPreview { UserDefaults.standard.set(self.voiceWakeMicName, forKey: voiceWakeMicNameKey) } } + } + + var voiceWakeLocaleID: String { + didSet { + self.ifNotPreview { + UserDefaults.standard.set(self.voiceWakeLocaleID, forKey: voiceWakeLocaleKey) + if self.swabbleEnabled { + Task { await VoiceWakeRuntime.shared.refresh(state: self) } + } + } + } + } + + var voiceWakeAdditionalLocaleIDs: [String] { + didSet { self.ifNotPreview { UserDefaults.standard.set( + self.voiceWakeAdditionalLocaleIDs, + forKey: voiceWakeAdditionalLocalesKey) } } + } + + var voicePushToTalkEnabled: Bool { + didSet { self.ifNotPreview { UserDefaults.standard.set( + self.voicePushToTalkEnabled, + forKey: voicePushToTalkEnabledKey) } } + } + + var talkEnabled: Bool { + didSet { + self.ifNotPreview { + UserDefaults.standard.set(self.talkEnabled, forKey: talkEnabledKey) + Task { await TalkModeController.shared.setEnabled(self.talkEnabled) } + } + } + } + + /// Gateway-provided UI accent color (hex). Optional; clients provide a default. + var seamColorHex: String? + + var iconOverride: IconOverrideSelection { + didSet { self.ifNotPreview { UserDefaults.standard.set(self.iconOverride.rawValue, forKey: iconOverrideKey) } } + } + + var isWorking: Bool = false + var earBoostActive: Bool = false + var blinkTick: Int = 0 + var sendCelebrationTick: Int = 0 + var heartbeatsEnabled: Bool { + didSet { + self.ifNotPreview { + UserDefaults.standard.set(self.heartbeatsEnabled, forKey: heartbeatsEnabledKey) + Task { _ = await GatewayConnection.shared.setHeartbeatsEnabled(self.heartbeatsEnabled) } + } + } + } + + var connectionMode: ConnectionMode { + didSet { + self.ifNotPreview { UserDefaults.standard.set(self.connectionMode.rawValue, forKey: connectionModeKey) } + self.syncGatewayConfigIfNeeded() + } + } + + var remoteTransport: RemoteTransport { + didSet { self.syncGatewayConfigIfNeeded() } + } + + var canvasEnabled: Bool { + didSet { self.ifNotPreview { UserDefaults.standard.set(self.canvasEnabled, forKey: canvasEnabledKey) } } + } + + var execApprovalMode: ExecApprovalQuickMode { + didSet { + self.ifNotPreview { + ExecApprovalsStore.updateDefaults { defaults in + defaults.security = self.execApprovalMode.security + defaults.ask = self.execApprovalMode.ask + } + } + } + } + + /// Tracks whether the Canvas panel is currently visible (not persisted). + var canvasPanelVisible: Bool = false + + var peekabooBridgeEnabled: Bool { + didSet { + self.ifNotPreview { + UserDefaults.standard.set(self.peekabooBridgeEnabled, forKey: peekabooBridgeEnabledKey) + Task { await PeekabooBridgeHostCoordinator.shared.setEnabled(self.peekabooBridgeEnabled) } + } + } + } + + var remoteTarget: String { + didSet { + self.ifNotPreview { UserDefaults.standard.set(self.remoteTarget, forKey: remoteTargetKey) } + self.syncGatewayConfigIfNeeded() + } + } + + var remoteUrl: String { + didSet { self.syncGatewayConfigIfNeeded() } + } + + var remoteIdentity: String { + didSet { self.ifNotPreview { UserDefaults.standard.set(self.remoteIdentity, forKey: remoteIdentityKey) } } + } + + var remoteProjectRoot: String { + didSet { self.ifNotPreview { UserDefaults.standard.set(self.remoteProjectRoot, forKey: remoteProjectRootKey) } } + } + + var remoteCliPath: String { + didSet { self.ifNotPreview { UserDefaults.standard.set(self.remoteCliPath, forKey: remoteCliPathKey) } } + } + + private var earBoostTask: Task? + + init(preview: Bool = false) { + let isPreview = preview || ProcessInfo.processInfo.isRunningTests + self.isPreview = isPreview + if !isPreview { + migrateLegacyDefaults() + } + let onboardingSeen = UserDefaults.standard.bool(forKey: onboardingSeenKey) + self.isPaused = UserDefaults.standard.bool(forKey: pauseDefaultsKey) + self.launchAtLogin = false + self.onboardingSeen = onboardingSeen + self.debugPaneEnabled = UserDefaults.standard.bool(forKey: debugPaneEnabledKey) + let savedVoiceWake = UserDefaults.standard.bool(forKey: swabbleEnabledKey) + self.swabbleEnabled = voiceWakeSupported ? savedVoiceWake : false + self.swabbleTriggerWords = UserDefaults.standard + .stringArray(forKey: swabbleTriggersKey) ?? defaultVoiceWakeTriggers + self.voiceWakeTriggerChime = Self.loadChime( + key: voiceWakeTriggerChimeKey, + fallback: .system(name: "Glass")) + self.voiceWakeSendChime = Self.loadChime( + key: voiceWakeSendChimeKey, + fallback: .system(name: "Glass")) + if let storedIconAnimations = UserDefaults.standard.object(forKey: iconAnimationsEnabledKey) as? Bool { + self.iconAnimationsEnabled = storedIconAnimations + } else { + self.iconAnimationsEnabled = true + UserDefaults.standard.set(true, forKey: iconAnimationsEnabledKey) + } + self.showDockIcon = UserDefaults.standard.bool(forKey: showDockIconKey) + self.voiceWakeMicID = UserDefaults.standard.string(forKey: voiceWakeMicKey) ?? "" + self.voiceWakeMicName = UserDefaults.standard.string(forKey: voiceWakeMicNameKey) ?? "" + self.voiceWakeLocaleID = UserDefaults.standard.string(forKey: voiceWakeLocaleKey) ?? Locale.current.identifier + self.voiceWakeAdditionalLocaleIDs = UserDefaults.standard + .stringArray(forKey: voiceWakeAdditionalLocalesKey) ?? [] + self.voicePushToTalkEnabled = UserDefaults.standard + .object(forKey: voicePushToTalkEnabledKey) as? Bool ?? false + self.talkEnabled = UserDefaults.standard.bool(forKey: talkEnabledKey) + self.seamColorHex = nil + if let storedHeartbeats = UserDefaults.standard.object(forKey: heartbeatsEnabledKey) as? Bool { + self.heartbeatsEnabled = storedHeartbeats + } else { + self.heartbeatsEnabled = true + UserDefaults.standard.set(true, forKey: heartbeatsEnabledKey) + } + if let storedOverride = UserDefaults.standard.string(forKey: iconOverrideKey), + let selection = IconOverrideSelection(rawValue: storedOverride) + { + self.iconOverride = selection + } else { + self.iconOverride = .system + UserDefaults.standard.set(IconOverrideSelection.system.rawValue, forKey: iconOverrideKey) + } + + let configRoot = OpenClawConfigFile.loadDict() + let configRemoteUrl = GatewayRemoteConfig.resolveUrlString(root: configRoot) + let configRemoteTransport = GatewayRemoteConfig.resolveTransport(root: configRoot) + let resolvedConnectionMode = ConnectionModeResolver.resolve(root: configRoot).mode + self.remoteTransport = configRemoteTransport + self.connectionMode = resolvedConnectionMode + + let storedRemoteTarget = UserDefaults.standard.string(forKey: remoteTargetKey) ?? "" + if resolvedConnectionMode == .remote, + configRemoteTransport != .direct, + storedRemoteTarget.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty, + let host = AppState.remoteHost(from: configRemoteUrl) + { + self.remoteTarget = "\(NSUserName())@\(host)" + } else { + self.remoteTarget = storedRemoteTarget + } + self.remoteUrl = configRemoteUrl ?? "" + self.remoteIdentity = UserDefaults.standard.string(forKey: remoteIdentityKey) ?? "" + self.remoteProjectRoot = UserDefaults.standard.string(forKey: remoteProjectRootKey) ?? "" + self.remoteCliPath = UserDefaults.standard.string(forKey: remoteCliPathKey) ?? "" + self.canvasEnabled = UserDefaults.standard.object(forKey: canvasEnabledKey) as? Bool ?? true + let execDefaults = ExecApprovalsStore.resolveDefaults() + self.execApprovalMode = ExecApprovalQuickMode.from(security: execDefaults.security, ask: execDefaults.ask) + self.peekabooBridgeEnabled = UserDefaults.standard + .object(forKey: peekabooBridgeEnabledKey) as? Bool ?? true + if !self.isPreview { + Task.detached(priority: .utility) { [weak self] in + let current = await LaunchAgentManager.status() + await MainActor.run { [weak self] in self?.launchAtLogin = current } + } + } + + if self.swabbleEnabled, !PermissionManager.voiceWakePermissionsGranted() { + self.swabbleEnabled = false + } + if self.talkEnabled, !PermissionManager.voiceWakePermissionsGranted() { + self.talkEnabled = false + } + + if !self.isPreview { + Task { await VoiceWakeRuntime.shared.refresh(state: self) } + Task { await TalkModeController.shared.setEnabled(self.talkEnabled) } + } + + self.isInitializing = false + if !self.isPreview { + self.startConfigWatcher() + } + } + + @MainActor + deinit { + self.configWatcher?.stop() + } + + private static func remoteHost(from urlString: String?) -> String? { + guard let raw = urlString?.trimmingCharacters(in: .whitespacesAndNewlines), + !raw.isEmpty, + let url = URL(string: raw), + let host = url.host?.trimmingCharacters(in: .whitespacesAndNewlines), + !host.isEmpty + else { + return nil + } + return host + } + + private static func sanitizeSSHTarget(_ value: String) -> String { + let trimmed = value.trimmingCharacters(in: .whitespacesAndNewlines) + if trimmed.hasPrefix("ssh ") { + return trimmed.replacingOccurrences(of: "ssh ", with: "") + .trimmingCharacters(in: .whitespacesAndNewlines) + } + return trimmed + } + + private func startConfigWatcher() { + let configUrl = OpenClawConfigFile.url() + self.configWatcher = ConfigFileWatcher(url: configUrl) { [weak self] in + Task { @MainActor in + self?.applyConfigFromDisk() + } + } + self.configWatcher?.start() + } + + private func applyConfigFromDisk() { + let root = OpenClawConfigFile.loadDict() + self.applyConfigOverrides(root) + } + + private func applyConfigOverrides(_ root: [String: Any]) { + let gateway = root["gateway"] as? [String: Any] + let modeRaw = (gateway?["mode"] as? String)?.trimmingCharacters(in: .whitespacesAndNewlines) + let remoteUrl = GatewayRemoteConfig.resolveUrlString(root: root) + let hasRemoteUrl = !(remoteUrl? + .trimmingCharacters(in: .whitespacesAndNewlines) + .isEmpty ?? true) + let remoteTransport = GatewayRemoteConfig.resolveTransport(root: root) + + let desiredMode: ConnectionMode? = switch modeRaw { + case "local": + .local + case "remote": + .remote + case "unconfigured": + .unconfigured + default: + nil + } + + if let desiredMode { + if desiredMode != self.connectionMode { + self.connectionMode = desiredMode + } + } else if hasRemoteUrl, self.connectionMode != .remote { + self.connectionMode = .remote + } + + if remoteTransport != self.remoteTransport { + self.remoteTransport = remoteTransport + } + let remoteUrlText = remoteUrl ?? "" + if remoteUrlText != self.remoteUrl { + self.remoteUrl = remoteUrlText + } + + let targetMode = desiredMode ?? self.connectionMode + if targetMode == .remote, + remoteTransport != .direct, + let host = AppState.remoteHost(from: remoteUrl) + { + self.updateRemoteTarget(host: host) + } + } + + private func updateRemoteTarget(host: String) { + let trimmed = self.remoteTarget.trimmingCharacters(in: .whitespacesAndNewlines) + guard let parsed = CommandResolver.parseSSHTarget(trimmed) else { return } + let trimmedUser = parsed.user?.trimmingCharacters(in: .whitespacesAndNewlines) + let user = (trimmedUser?.isEmpty ?? true) ? nil : trimmedUser + let port = parsed.port + let assembled: String + if let user { + assembled = port == 22 ? "\(user)@\(host)" : "\(user)@\(host):\(port)" + } else { + assembled = port == 22 ? host : "\(host):\(port)" + } + if assembled != self.remoteTarget { + self.remoteTarget = assembled + } + } + + private func syncGatewayConfigIfNeeded() { + guard !self.isPreview, !self.isInitializing else { return } + + let connectionMode = self.connectionMode + let remoteTarget = self.remoteTarget + let remoteIdentity = self.remoteIdentity + let remoteTransport = self.remoteTransport + let remoteUrl = self.remoteUrl + let desiredMode: String? = switch connectionMode { + case .local: + "local" + case .remote: + "remote" + case .unconfigured: + nil + } + let remoteHost = connectionMode == .remote + ? CommandResolver.parseSSHTarget(remoteTarget)?.host + : nil + + Task { @MainActor in + // Keep app-only connection settings local to avoid overwriting remote gateway config. + var root = OpenClawConfigFile.loadDict() + var gateway = root["gateway"] as? [String: Any] ?? [:] + var changed = false + + let currentMode = (gateway["mode"] as? String)?.trimmingCharacters(in: .whitespacesAndNewlines) + if let desiredMode { + if currentMode != desiredMode { + gateway["mode"] = desiredMode + changed = true + } + } else if currentMode != nil { + gateway.removeValue(forKey: "mode") + changed = true + } + + if connectionMode == .remote { + var remote = gateway["remote"] as? [String: Any] ?? [:] + var remoteChanged = false + + if remoteTransport == .direct { + let trimmedUrl = remoteUrl.trimmingCharacters(in: .whitespacesAndNewlines) + if trimmedUrl.isEmpty { + if remote["url"] != nil { + remote.removeValue(forKey: "url") + remoteChanged = true + } + } else { + let normalizedUrl = GatewayRemoteConfig.normalizeGatewayUrlString(trimmedUrl) ?? trimmedUrl + if (remote["url"] as? String) != normalizedUrl { + remote["url"] = normalizedUrl + remoteChanged = true + } + } + if (remote["transport"] as? String) != RemoteTransport.direct.rawValue { + remote["transport"] = RemoteTransport.direct.rawValue + remoteChanged = true + } + } else { + if remote["transport"] != nil { + remote.removeValue(forKey: "transport") + remoteChanged = true + } + if let host = remoteHost { + let existingUrl = (remote["url"] as? String)? + .trimmingCharacters(in: .whitespacesAndNewlines) ?? "" + let parsedExisting = existingUrl.isEmpty ? nil : URL(string: existingUrl) + let scheme = parsedExisting?.scheme?.isEmpty == false ? parsedExisting?.scheme : "ws" + let port = parsedExisting?.port ?? 18789 + let desiredUrl = "\(scheme ?? "ws")://\(host):\(port)" + if existingUrl != desiredUrl { + remote["url"] = desiredUrl + remoteChanged = true + } + } + + let sanitizedTarget = Self.sanitizeSSHTarget(remoteTarget) + if !sanitizedTarget.isEmpty { + if (remote["sshTarget"] as? String) != sanitizedTarget { + remote["sshTarget"] = sanitizedTarget + remoteChanged = true + } + } else if remote["sshTarget"] != nil { + remote.removeValue(forKey: "sshTarget") + remoteChanged = true + } + + let trimmedIdentity = remoteIdentity.trimmingCharacters(in: .whitespacesAndNewlines) + if !trimmedIdentity.isEmpty { + if (remote["sshIdentity"] as? String) != trimmedIdentity { + remote["sshIdentity"] = trimmedIdentity + remoteChanged = true + } + } else if remote["sshIdentity"] != nil { + remote.removeValue(forKey: "sshIdentity") + remoteChanged = true + } + } + + if remoteChanged { + gateway["remote"] = remote + changed = true + } + } + + guard changed else { return } + if gateway.isEmpty { + root.removeValue(forKey: "gateway") + } else { + root["gateway"] = gateway + } + OpenClawConfigFile.saveDict(root) + } + } + + func triggerVoiceEars(ttl: TimeInterval? = 5) { + self.earBoostTask?.cancel() + self.earBoostActive = true + + guard let ttl else { return } + + self.earBoostTask = Task { [weak self] in + try? await Task.sleep(nanoseconds: UInt64(ttl * 1_000_000_000)) + await MainActor.run { [weak self] in self?.earBoostActive = false } + } + } + + func stopVoiceEars() { + self.earBoostTask?.cancel() + self.earBoostTask = nil + self.earBoostActive = false + } + + func blinkOnce() { + self.blinkTick &+= 1 + } + + func celebrateSend() { + self.sendCelebrationTick &+= 1 + } + + func setVoiceWakeEnabled(_ enabled: Bool) async { + guard voiceWakeSupported else { + self.swabbleEnabled = false + return + } + + self.swabbleEnabled = enabled + guard !self.isPreview else { return } + + if !enabled { + Task { await VoiceWakeRuntime.shared.refresh(state: self) } + return + } + + if PermissionManager.voiceWakePermissionsGranted() { + Task { await VoiceWakeRuntime.shared.refresh(state: self) } + return + } + + let granted = await PermissionManager.ensureVoiceWakePermissions(interactive: true) + self.swabbleEnabled = granted + Task { await VoiceWakeRuntime.shared.refresh(state: self) } + } + + func setTalkEnabled(_ enabled: Bool) async { + guard voiceWakeSupported else { + self.talkEnabled = false + await GatewayConnection.shared.talkMode(enabled: false, phase: "disabled") + return + } + + self.talkEnabled = enabled + guard !self.isPreview else { return } + + if !enabled { + await GatewayConnection.shared.talkMode(enabled: false, phase: "disabled") + return + } + + if PermissionManager.voiceWakePermissionsGranted() { + await GatewayConnection.shared.talkMode(enabled: true, phase: "enabled") + return + } + + let granted = await PermissionManager.ensureVoiceWakePermissions(interactive: true) + self.talkEnabled = granted + await GatewayConnection.shared.talkMode(enabled: granted, phase: granted ? "enabled" : "denied") + } + + // MARK: - Global wake words sync (Gateway-owned) + + func applyGlobalVoiceWakeTriggers(_ triggers: [String]) { + self.suppressVoiceWakeGlobalSync = true + self.swabbleTriggerWords = triggers + self.suppressVoiceWakeGlobalSync = false + } + + private func scheduleVoiceWakeGlobalSyncIfNeeded() { + guard !self.suppressVoiceWakeGlobalSync else { return } + let sanitized = sanitizeVoiceWakeTriggers(self.swabbleTriggerWords) + self.voiceWakeGlobalSyncTask?.cancel() + self.voiceWakeGlobalSyncTask = Task { [sanitized] in + try? await Task.sleep(nanoseconds: 650_000_000) + await GatewayConnection.shared.voiceWakeSetTriggers(sanitized) + } + } + + func setWorking(_ working: Bool) { + self.isWorking = working + } + + // MARK: - Chime persistence + + private static func loadChime(key: String, fallback: VoiceWakeChime) -> VoiceWakeChime { + guard let data = UserDefaults.standard.data(forKey: key) else { return fallback } + if let decoded = try? JSONDecoder().decode(VoiceWakeChime.self, from: data) { + return decoded + } + return fallback + } + + private func storeChime(_ chime: VoiceWakeChime, key: String) { + guard let data = try? JSONEncoder().encode(chime) else { return } + UserDefaults.standard.set(data, forKey: key) + } +} + +extension AppState { + static var preview: AppState { + let state = AppState(preview: true) + state.isPaused = false + state.launchAtLogin = true + state.onboardingSeen = true + state.debugPaneEnabled = true + state.swabbleEnabled = true + state.swabbleTriggerWords = ["Claude", "Computer", "Jarvis"] + state.voiceWakeTriggerChime = .system(name: "Glass") + state.voiceWakeSendChime = .system(name: "Ping") + state.iconAnimationsEnabled = true + state.showDockIcon = true + state.voiceWakeMicID = "BuiltInMic" + state.voiceWakeMicName = "Built-in Microphone" + state.voiceWakeLocaleID = Locale.current.identifier + state.voiceWakeAdditionalLocaleIDs = ["en-US", "de-DE"] + state.voicePushToTalkEnabled = false + state.talkEnabled = false + state.iconOverride = .system + state.heartbeatsEnabled = true + state.connectionMode = .local + state.remoteTransport = .ssh + state.canvasEnabled = true + state.remoteTarget = "user@example.com" + state.remoteUrl = "wss://gateway.example.ts.net" + state.remoteIdentity = "~/.ssh/id_ed25519" + state.remoteProjectRoot = "~/Projects/openclaw" + state.remoteCliPath = "" + return state + } +} + +@MainActor +enum AppStateStore { + static let shared = AppState() + static var isPausedFlag: Bool { UserDefaults.standard.bool(forKey: pauseDefaultsKey) } + + static func updateLaunchAtLogin(enabled: Bool) { + Task.detached(priority: .utility) { + await LaunchAgentManager.set(enabled: enabled, bundlePath: Bundle.main.bundlePath) + } + } + + static var canvasEnabled: Bool { + UserDefaults.standard.object(forKey: canvasEnabledKey) as? Bool ?? true + } +} + +@MainActor +enum AppActivationPolicy { + static func apply(showDockIcon: Bool) { + _ = showDockIcon + DockIconManager.shared.updateDockVisibility() + } +} diff --git a/apps/macos/Sources/OpenClaw/AudioInputDeviceObserver.swift b/apps/macos/Sources/OpenClaw/AudioInputDeviceObserver.swift new file mode 100644 index 0000000000000000000000000000000000000000..abbddb245887153e2b2ae1690592b410f21fdad7 --- /dev/null +++ b/apps/macos/Sources/OpenClaw/AudioInputDeviceObserver.swift @@ -0,0 +1,216 @@ +import CoreAudio +import Foundation +import OSLog + +final class AudioInputDeviceObserver { + private let logger = Logger(subsystem: "ai.openclaw", category: "audio.devices") + private var isActive = false + private var devicesListener: AudioObjectPropertyListenerBlock? + private var defaultInputListener: AudioObjectPropertyListenerBlock? + + static func defaultInputDeviceUID() -> String? { + let systemObject = AudioObjectID(kAudioObjectSystemObject) + var address = AudioObjectPropertyAddress( + mSelector: kAudioHardwarePropertyDefaultInputDevice, + mScope: kAudioObjectPropertyScopeGlobal, + mElement: kAudioObjectPropertyElementMain) + var deviceID = AudioObjectID(0) + var size = UInt32(MemoryLayout.size) + let status = AudioObjectGetPropertyData( + systemObject, + &address, + 0, + nil, + &size, + &deviceID) + guard status == noErr, deviceID != 0 else { return nil } + return self.deviceUID(for: deviceID) + } + + static func aliveInputDeviceUIDs() -> Set { + let systemObject = AudioObjectID(kAudioObjectSystemObject) + var address = AudioObjectPropertyAddress( + mSelector: kAudioHardwarePropertyDevices, + mScope: kAudioObjectPropertyScopeGlobal, + mElement: kAudioObjectPropertyElementMain) + var size: UInt32 = 0 + var status = AudioObjectGetPropertyDataSize(systemObject, &address, 0, nil, &size) + guard status == noErr, size > 0 else { return [] } + + let count = Int(size) / MemoryLayout.size + var deviceIDs = [AudioObjectID](repeating: 0, count: count) + status = AudioObjectGetPropertyData(systemObject, &address, 0, nil, &size, &deviceIDs) + guard status == noErr else { return [] } + + var output = Set() + for deviceID in deviceIDs { + guard self.deviceIsAlive(deviceID) else { continue } + guard self.deviceHasInput(deviceID) else { continue } + if let uid = self.deviceUID(for: deviceID) { + output.insert(uid) + } + } + return output + } + + static func defaultInputDeviceSummary() -> String { + let systemObject = AudioObjectID(kAudioObjectSystemObject) + var address = AudioObjectPropertyAddress( + mSelector: kAudioHardwarePropertyDefaultInputDevice, + mScope: kAudioObjectPropertyScopeGlobal, + mElement: kAudioObjectPropertyElementMain) + var deviceID = AudioObjectID(0) + var size = UInt32(MemoryLayout.size) + let status = AudioObjectGetPropertyData( + systemObject, + &address, + 0, + nil, + &size, + &deviceID) + guard status == noErr, deviceID != 0 else { + return "defaultInput=unknown" + } + let uid = self.deviceUID(for: deviceID) ?? "unknown" + let name = self.deviceName(for: deviceID) ?? "unknown" + return "defaultInput=\(name) (\(uid))" + } + + func start(onChange: @escaping @Sendable () -> Void) { + guard !self.isActive else { return } + self.isActive = true + + let systemObject = AudioObjectID(kAudioObjectSystemObject) + let queue = DispatchQueue.main + + var devicesAddress = AudioObjectPropertyAddress( + mSelector: kAudioHardwarePropertyDevices, + mScope: kAudioObjectPropertyScopeGlobal, + mElement: kAudioObjectPropertyElementMain) + let devicesListener: AudioObjectPropertyListenerBlock = { _, _ in + self.logDefaultInputChange(reason: "devices") + onChange() + } + let devicesStatus = AudioObjectAddPropertyListenerBlock( + systemObject, + &devicesAddress, + queue, + devicesListener) + + var defaultInputAddress = AudioObjectPropertyAddress( + mSelector: kAudioHardwarePropertyDefaultInputDevice, + mScope: kAudioObjectPropertyScopeGlobal, + mElement: kAudioObjectPropertyElementMain) + let defaultInputListener: AudioObjectPropertyListenerBlock = { _, _ in + self.logDefaultInputChange(reason: "default") + onChange() + } + let defaultStatus = AudioObjectAddPropertyListenerBlock( + systemObject, + &defaultInputAddress, + queue, + defaultInputListener) + + if devicesStatus != noErr || defaultStatus != noErr { + self.logger.error("audio device observer install failed devices=\(devicesStatus) default=\(defaultStatus)") + } + + self.logger.info("audio device observer started (\(Self.defaultInputDeviceSummary(), privacy: .public))") + + self.devicesListener = devicesListener + self.defaultInputListener = defaultInputListener + } + + func stop() { + guard self.isActive else { return } + self.isActive = false + let systemObject = AudioObjectID(kAudioObjectSystemObject) + + if let devicesListener { + var devicesAddress = AudioObjectPropertyAddress( + mSelector: kAudioHardwarePropertyDevices, + mScope: kAudioObjectPropertyScopeGlobal, + mElement: kAudioObjectPropertyElementMain) + _ = AudioObjectRemovePropertyListenerBlock( + systemObject, + &devicesAddress, + DispatchQueue.main, + devicesListener) + } + + if let defaultInputListener { + var defaultInputAddress = AudioObjectPropertyAddress( + mSelector: kAudioHardwarePropertyDefaultInputDevice, + mScope: kAudioObjectPropertyScopeGlobal, + mElement: kAudioObjectPropertyElementMain) + _ = AudioObjectRemovePropertyListenerBlock( + systemObject, + &defaultInputAddress, + DispatchQueue.main, + defaultInputListener) + } + + self.devicesListener = nil + self.defaultInputListener = nil + } + + private static func deviceUID(for deviceID: AudioObjectID) -> String? { + var address = AudioObjectPropertyAddress( + mSelector: kAudioDevicePropertyDeviceUID, + mScope: kAudioObjectPropertyScopeGlobal, + mElement: kAudioObjectPropertyElementMain) + var uid: Unmanaged? + var size = UInt32(MemoryLayout?>.size) + let status = AudioObjectGetPropertyData(deviceID, &address, 0, nil, &size, &uid) + guard status == noErr, let uid else { return nil } + return uid.takeUnretainedValue() as String + } + + private static func deviceName(for deviceID: AudioObjectID) -> String? { + var address = AudioObjectPropertyAddress( + mSelector: kAudioObjectPropertyName, + mScope: kAudioObjectPropertyScopeGlobal, + mElement: kAudioObjectPropertyElementMain) + var name: Unmanaged? + var size = UInt32(MemoryLayout?>.size) + let status = AudioObjectGetPropertyData(deviceID, &address, 0, nil, &size, &name) + guard status == noErr, let name else { return nil } + return name.takeUnretainedValue() as String + } + + private static func deviceIsAlive(_ deviceID: AudioObjectID) -> Bool { + var address = AudioObjectPropertyAddress( + mSelector: kAudioDevicePropertyDeviceIsAlive, + mScope: kAudioObjectPropertyScopeGlobal, + mElement: kAudioObjectPropertyElementMain) + var alive: UInt32 = 0 + var size = UInt32(MemoryLayout.size) + let status = AudioObjectGetPropertyData(deviceID, &address, 0, nil, &size, &alive) + return status == noErr && alive != 0 + } + + private static func deviceHasInput(_ deviceID: AudioObjectID) -> Bool { + var address = AudioObjectPropertyAddress( + mSelector: kAudioDevicePropertyStreamConfiguration, + mScope: kAudioDevicePropertyScopeInput, + mElement: kAudioObjectPropertyElementMain) + var size: UInt32 = 0 + var status = AudioObjectGetPropertyDataSize(deviceID, &address, 0, nil, &size) + guard status == noErr, size > 0 else { return false } + + let raw = UnsafeMutableRawPointer.allocate( + byteCount: Int(size), + alignment: MemoryLayout.alignment) + defer { raw.deallocate() } + let bufferList = raw.bindMemory(to: AudioBufferList.self, capacity: 1) + status = AudioObjectGetPropertyData(deviceID, &address, 0, nil, &size, bufferList) + guard status == noErr else { return false } + + let buffers = UnsafeMutableAudioBufferListPointer(bufferList) + return buffers.contains(where: { $0.mNumberChannels > 0 }) + } + + private func logDefaultInputChange(reason: StaticString) { + self.logger.info("audio input changed (\(reason)) (\(Self.defaultInputDeviceSummary(), privacy: .public))") + } +} diff --git a/apps/macos/Sources/OpenClaw/CLIInstallPrompter.swift b/apps/macos/Sources/OpenClaw/CLIInstallPrompter.swift new file mode 100644 index 0000000000000000000000000000000000000000..482f36fd6d0c7fb93814a565ade599b02f25e60c --- /dev/null +++ b/apps/macos/Sources/OpenClaw/CLIInstallPrompter.swift @@ -0,0 +1,84 @@ +import AppKit +import Foundation +import OSLog + +@MainActor +final class CLIInstallPrompter { + static let shared = CLIInstallPrompter() + private let logger = Logger(subsystem: "ai.openclaw", category: "cli.prompt") + private var isPrompting = false + + func checkAndPromptIfNeeded(reason: String) { + guard self.shouldPrompt() else { return } + guard let version = Self.appVersion() else { return } + self.isPrompting = true + UserDefaults.standard.set(version, forKey: cliInstallPromptedVersionKey) + + let alert = NSAlert() + alert.messageText = "Install OpenClaw CLI?" + alert.informativeText = "Local mode needs the CLI so launchd can run the gateway." + alert.addButton(withTitle: "Install CLI") + alert.addButton(withTitle: "Not now") + alert.addButton(withTitle: "Open Settings") + let response = alert.runModal() + + switch response { + case .alertFirstButtonReturn: + Task { await self.installCLI() } + case .alertThirdButtonReturn: + self.openSettings(tab: .general) + default: + break + } + + self.logger.debug("cli install prompt handled reason=\(reason, privacy: .public)") + self.isPrompting = false + } + + private func shouldPrompt() -> Bool { + guard !self.isPrompting else { return false } + guard AppStateStore.shared.onboardingSeen else { return false } + guard AppStateStore.shared.connectionMode == .local else { return false } + guard CLIInstaller.installedLocation() == nil else { return false } + guard let version = Self.appVersion() else { return false } + let lastPrompt = UserDefaults.standard.string(forKey: cliInstallPromptedVersionKey) + return lastPrompt != version + } + + private func installCLI() async { + let status = StatusBox() + await CLIInstaller.install { message in + await status.set(message) + } + if let message = await status.get() { + let alert = NSAlert() + alert.messageText = "CLI install finished" + alert.informativeText = message + alert.runModal() + } + } + + private func openSettings(tab: SettingsTab) { + SettingsTabRouter.request(tab) + SettingsWindowOpener.shared.open() + DispatchQueue.main.async { + NotificationCenter.default.post(name: .openclawSelectSettingsTab, object: tab) + } + } + + private static func appVersion() -> String? { + Bundle.main.infoDictionary?["CFBundleShortVersionString"] as? String + } +} + +private actor StatusBox { + private var value: String? + + func set(_ value: String) { + self.value = value + } + + func get() -> String? { + self.value + } +} diff --git a/apps/macos/Sources/OpenClaw/CLIInstaller.swift b/apps/macos/Sources/OpenClaw/CLIInstaller.swift new file mode 100644 index 0000000000000000000000000000000000000000..ce6d25202ae22c161818d4a1d9ef1e9598dc5e02 --- /dev/null +++ b/apps/macos/Sources/OpenClaw/CLIInstaller.swift @@ -0,0 +1,103 @@ +import Foundation + +@MainActor +enum CLIInstaller { + static func installedLocation() -> String? { + self.installedLocation( + searchPaths: CommandResolver.preferredPaths(), + fileManager: .default) + } + + static func installedLocation( + searchPaths: [String], + fileManager: FileManager) -> String? + { + for basePath in searchPaths { + let candidate = URL(fileURLWithPath: basePath).appendingPathComponent("openclaw").path + var isDirectory: ObjCBool = false + + guard fileManager.fileExists(atPath: candidate, isDirectory: &isDirectory), + !isDirectory.boolValue + else { + continue + } + + guard fileManager.isExecutableFile(atPath: candidate) else { continue } + + return candidate + } + + return nil + } + + static func isInstalled() -> Bool { + self.installedLocation() != nil + } + + static func install(statusHandler: @escaping @MainActor @Sendable (String) async -> Void) async { + let expected = GatewayEnvironment.expectedGatewayVersionString() ?? "latest" + let prefix = Self.installPrefix() + await statusHandler("Installing openclaw CLI…") + let cmd = self.installScriptCommand(version: expected, prefix: prefix) + let response = await ShellExecutor.runDetailed(command: cmd, cwd: nil, env: nil, timeout: 900) + + if response.success { + let parsed = self.parseInstallEvents(response.stdout) + let installedVersion = parsed.last { $0.event == "done" }?.version + let summary = installedVersion.map { "Installed openclaw \($0)." } ?? "Installed openclaw." + await statusHandler(summary) + return + } + + let parsed = self.parseInstallEvents(response.stdout) + if let error = parsed.last(where: { $0.event == "error" })?.message { + await statusHandler("Install failed: \(error)") + return + } + + let detail = response.stderr.trimmingCharacters(in: .whitespacesAndNewlines) + let fallback = response.errorMessage ?? "install failed" + await statusHandler("Install failed: \(detail.isEmpty ? fallback : detail)") + } + + private static func installPrefix() -> String { + FileManager().homeDirectoryForCurrentUser + .appendingPathComponent(".openclaw") + .path + } + + private static func installScriptCommand(version: String, prefix: String) -> [String] { + let escapedVersion = self.shellEscape(version) + let escapedPrefix = self.shellEscape(prefix) + let script = """ + curl -fsSL https://openclaw.bot/install-cli.sh | \ + bash -s -- --json --no-onboard --prefix \(escapedPrefix) --version \(escapedVersion) + """ + return ["/bin/bash", "-lc", script] + } + + private static func parseInstallEvents(_ output: String) -> [InstallEvent] { + let decoder = JSONDecoder() + let lines = output + .split(whereSeparator: \.isNewline) + .map { String($0) } + var events: [InstallEvent] = [] + for line in lines { + guard let data = line.data(using: .utf8) else { continue } + if let event = try? decoder.decode(InstallEvent.self, from: data) { + events.append(event) + } + } + return events + } + + private static func shellEscape(_ raw: String) -> String { + "'" + raw.replacingOccurrences(of: "'", with: "'\"'\"'") + "'" + } +} + +private struct InstallEvent: Decodable { + let event: String + let version: String? + let message: String? +} diff --git a/apps/macos/Sources/OpenClaw/CameraCaptureService.swift b/apps/macos/Sources/OpenClaw/CameraCaptureService.swift new file mode 100644 index 0000000000000000000000000000000000000000..8653b05dcbb2a258f25442a8fd8eeeda0692e482 --- /dev/null +++ b/apps/macos/Sources/OpenClaw/CameraCaptureService.swift @@ -0,0 +1,425 @@ +import AVFoundation +import OpenClawIPC +import OpenClawKit +import CoreGraphics +import Foundation +import OSLog + +actor CameraCaptureService { + struct CameraDeviceInfo: Encodable, Sendable { + let id: String + let name: String + let position: String + let deviceType: String + } + + enum CameraError: LocalizedError, Sendable { + case cameraUnavailable + case microphoneUnavailable + case permissionDenied(kind: String) + case captureFailed(String) + case exportFailed(String) + + var errorDescription: String? { + switch self { + case .cameraUnavailable: + "Camera unavailable" + case .microphoneUnavailable: + "Microphone unavailable" + case let .permissionDenied(kind): + "\(kind) permission denied" + case let .captureFailed(msg): + msg + case let .exportFailed(msg): + msg + } + } + } + + private let logger = Logger(subsystem: "ai.openclaw", category: "camera") + + func listDevices() -> [CameraDeviceInfo] { + Self.availableCameras().map { device in + CameraDeviceInfo( + id: device.uniqueID, + name: device.localizedName, + position: Self.positionLabel(device.position), + deviceType: device.deviceType.rawValue) + } + } + + func snap( + facing: CameraFacing?, + maxWidth: Int?, + quality: Double?, + deviceId: String?, + delayMs: Int) async throws -> (data: Data, size: CGSize) + { + let facing = facing ?? .front + let normalized = Self.normalizeSnap(maxWidth: maxWidth, quality: quality) + let maxWidth = normalized.maxWidth + let quality = normalized.quality + let delayMs = max(0, delayMs) + let deviceId = deviceId?.trimmingCharacters(in: .whitespacesAndNewlines) + + try await self.ensureAccess(for: .video) + + let session = AVCaptureSession() + session.sessionPreset = .photo + + guard let device = Self.pickCamera(facing: facing, deviceId: deviceId) else { + throw CameraError.cameraUnavailable + } + + let input = try AVCaptureDeviceInput(device: device) + guard session.canAddInput(input) else { + throw CameraError.captureFailed("Failed to add camera input") + } + session.addInput(input) + + let output = AVCapturePhotoOutput() + guard session.canAddOutput(output) else { + throw CameraError.captureFailed("Failed to add photo output") + } + session.addOutput(output) + output.maxPhotoQualityPrioritization = .quality + + session.startRunning() + defer { session.stopRunning() } + await Self.warmUpCaptureSession() + await self.waitForExposureAndWhiteBalance(device: device) + await self.sleepDelayMs(delayMs) + + let settings: AVCapturePhotoSettings = { + if output.availablePhotoCodecTypes.contains(.jpeg) { + return AVCapturePhotoSettings(format: [AVVideoCodecKey: AVVideoCodecType.jpeg]) + } + return AVCapturePhotoSettings() + }() + settings.photoQualityPrioritization = .quality + + var delegate: PhotoCaptureDelegate? + let rawData: Data = try await withCheckedThrowingContinuation { cont in + let d = PhotoCaptureDelegate(cont) + delegate = d + output.capturePhoto(with: settings, delegate: d) + } + withExtendedLifetime(delegate) {} + + let maxPayloadBytes = 5 * 1024 * 1024 + // Base64 inflates payloads by ~4/3; cap encoded bytes so the payload stays under 5MB (API limit). + let maxEncodedBytes = (maxPayloadBytes / 4) * 3 + let res = try JPEGTranscoder.transcodeToJPEG( + imageData: rawData, + maxWidthPx: maxWidth, + quality: quality, + maxBytes: maxEncodedBytes) + return (data: res.data, size: CGSize(width: res.widthPx, height: res.heightPx)) + } + + func clip( + facing: CameraFacing?, + durationMs: Int?, + includeAudio: Bool, + deviceId: String?, + outPath: String?) async throws -> (path: String, durationMs: Int, hasAudio: Bool) + { + let facing = facing ?? .front + let durationMs = Self.clampDurationMs(durationMs) + let deviceId = deviceId?.trimmingCharacters(in: .whitespacesAndNewlines) + + try await self.ensureAccess(for: .video) + if includeAudio { + try await self.ensureAccess(for: .audio) + } + + let session = AVCaptureSession() + session.sessionPreset = .high + + guard let camera = Self.pickCamera(facing: facing, deviceId: deviceId) else { + throw CameraError.cameraUnavailable + } + let cameraInput = try AVCaptureDeviceInput(device: camera) + guard session.canAddInput(cameraInput) else { + throw CameraError.captureFailed("Failed to add camera input") + } + session.addInput(cameraInput) + + if includeAudio { + guard let mic = AVCaptureDevice.default(for: .audio) else { + throw CameraError.microphoneUnavailable + } + let micInput = try AVCaptureDeviceInput(device: mic) + guard session.canAddInput(micInput) else { + throw CameraError.captureFailed("Failed to add microphone input") + } + session.addInput(micInput) + } + + let output = AVCaptureMovieFileOutput() + guard session.canAddOutput(output) else { + throw CameraError.captureFailed("Failed to add movie output") + } + session.addOutput(output) + output.maxRecordedDuration = CMTime(value: Int64(durationMs), timescale: 1000) + + session.startRunning() + defer { session.stopRunning() } + await Self.warmUpCaptureSession() + + let tmpMovURL = FileManager().temporaryDirectory + .appendingPathComponent("openclaw-camera-\(UUID().uuidString).mov") + defer { try? FileManager().removeItem(at: tmpMovURL) } + + let outputURL: URL = { + if let outPath, !outPath.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty { + return URL(fileURLWithPath: outPath) + } + return FileManager().temporaryDirectory + .appendingPathComponent("openclaw-camera-\(UUID().uuidString).mp4") + }() + + // Ensure we don't fail exporting due to an existing file. + try? FileManager().removeItem(at: outputURL) + + let logger = self.logger + var delegate: MovieFileDelegate? + let recordedURL: URL = try await withCheckedThrowingContinuation { cont in + let d = MovieFileDelegate(cont, logger: logger) + delegate = d + output.startRecording(to: tmpMovURL, recordingDelegate: d) + } + withExtendedLifetime(delegate) {} + + try await Self.exportToMP4(inputURL: recordedURL, outputURL: outputURL) + return (path: outputURL.path, durationMs: durationMs, hasAudio: includeAudio) + } + + private func ensureAccess(for mediaType: AVMediaType) async throws { + let status = AVCaptureDevice.authorizationStatus(for: mediaType) + switch status { + case .authorized: + return + case .notDetermined: + let ok = await withCheckedContinuation(isolation: nil) { cont in + AVCaptureDevice.requestAccess(for: mediaType) { granted in + cont.resume(returning: granted) + } + } + if !ok { + throw CameraError.permissionDenied(kind: mediaType == .video ? "Camera" : "Microphone") + } + case .denied, .restricted: + throw CameraError.permissionDenied(kind: mediaType == .video ? "Camera" : "Microphone") + @unknown default: + throw CameraError.permissionDenied(kind: mediaType == .video ? "Camera" : "Microphone") + } + } + + private nonisolated static func availableCameras() -> [AVCaptureDevice] { + var types: [AVCaptureDevice.DeviceType] = [ + .builtInWideAngleCamera, + .continuityCamera, + ] + if let external = externalDeviceType() { + types.append(external) + } + let session = AVCaptureDevice.DiscoverySession( + deviceTypes: types, + mediaType: .video, + position: .unspecified) + return session.devices + } + + private nonisolated static func externalDeviceType() -> AVCaptureDevice.DeviceType? { + if #available(macOS 14.0, *) { + return .external + } + // Use raw value to avoid deprecated symbol in the SDK. + return AVCaptureDevice.DeviceType(rawValue: "AVCaptureDeviceTypeExternalUnknown") + } + + private nonisolated static func pickCamera( + facing: CameraFacing, + deviceId: String?) -> AVCaptureDevice? + { + if let deviceId, !deviceId.isEmpty { + if let match = availableCameras().first(where: { $0.uniqueID == deviceId }) { + return match + } + } + let position: AVCaptureDevice.Position = (facing == .front) ? .front : .back + + if let device = AVCaptureDevice.default(.builtInWideAngleCamera, for: .video, position: position) { + return device + } + + // Many macOS cameras report `unspecified` position; fall back to any default. + return AVCaptureDevice.default(for: .video) + } + + private nonisolated static func clampQuality(_ quality: Double?) -> Double { + let q = quality ?? 0.9 + return min(1.0, max(0.05, q)) + } + + nonisolated static func normalizeSnap(maxWidth: Int?, quality: Double?) -> (maxWidth: Int, quality: Double) { + // Default to a reasonable max width to keep downstream payload sizes manageable. + // If you need full-res, explicitly request a larger maxWidth. + let maxWidth = maxWidth.flatMap { $0 > 0 ? $0 : nil } ?? 1600 + let quality = Self.clampQuality(quality) + return (maxWidth: maxWidth, quality: quality) + } + + private nonisolated static func clampDurationMs(_ ms: Int?) -> Int { + let v = ms ?? 3000 + return min(60000, max(250, v)) + } + + private nonisolated static func exportToMP4(inputURL: URL, outputURL: URL) async throws { + let asset = AVURLAsset(url: inputURL) + guard let export = AVAssetExportSession(asset: asset, presetName: AVAssetExportPresetMediumQuality) else { + throw CameraError.exportFailed("Failed to create export session") + } + export.shouldOptimizeForNetworkUse = true + + if #available(macOS 15.0, *) { + do { + try await export.export(to: outputURL, as: .mp4) + return + } catch { + throw CameraError.exportFailed(error.localizedDescription) + } + } else { + export.outputURL = outputURL + export.outputFileType = .mp4 + + try await withCheckedThrowingContinuation(isolation: nil) { (cont: CheckedContinuation) in + export.exportAsynchronously { + cont.resume(returning: ()) + } + } + + switch export.status { + case .completed: + return + case .failed: + throw CameraError.exportFailed(export.error?.localizedDescription ?? "export failed") + case .cancelled: + throw CameraError.exportFailed("export cancelled") + default: + throw CameraError.exportFailed("export did not complete (\(export.status.rawValue))") + } + } + } + + private nonisolated static func warmUpCaptureSession() async { + // A short delay after `startRunning()` significantly reduces "blank first frame" captures on some devices. + try? await Task.sleep(nanoseconds: 150_000_000) // 150ms + } + + private func waitForExposureAndWhiteBalance(device: AVCaptureDevice) async { + let stepNs: UInt64 = 50_000_000 + let maxSteps = 30 // ~1.5s + for _ in 0.. 0 else { return } + let ns = UInt64(min(delayMs, 10000)) * 1_000_000 + try? await Task.sleep(nanoseconds: ns) + } + + private nonisolated static func positionLabel(_ position: AVCaptureDevice.Position) -> String { + switch position { + case .front: "front" + case .back: "back" + default: "unspecified" + } + } +} + +private final class PhotoCaptureDelegate: NSObject, AVCapturePhotoCaptureDelegate { + private var cont: CheckedContinuation? + private var didResume = false + + init(_ cont: CheckedContinuation) { + self.cont = cont + } + + func photoOutput( + _ output: AVCapturePhotoOutput, + didFinishProcessingPhoto photo: AVCapturePhoto, + error: Error?) + { + guard !self.didResume, let cont else { return } + self.didResume = true + self.cont = nil + if let error { + cont.resume(throwing: error) + return + } + guard let data = photo.fileDataRepresentation() else { + cont.resume(throwing: CameraCaptureService.CameraError.captureFailed("No photo data")) + return + } + if data.isEmpty { + cont.resume(throwing: CameraCaptureService.CameraError.captureFailed("Photo data empty")) + return + } + cont.resume(returning: data) + } + + func photoOutput( + _ output: AVCapturePhotoOutput, + didFinishCaptureFor resolvedSettings: AVCaptureResolvedPhotoSettings, + error: Error?) + { + guard let error else { return } + guard !self.didResume, let cont else { return } + self.didResume = true + self.cont = nil + cont.resume(throwing: error) + } +} + +private final class MovieFileDelegate: NSObject, AVCaptureFileOutputRecordingDelegate { + private var cont: CheckedContinuation? + private let logger: Logger + + init(_ cont: CheckedContinuation, logger: Logger) { + self.cont = cont + self.logger = logger + } + + func fileOutput( + _ output: AVCaptureFileOutput, + didFinishRecordingTo outputFileURL: URL, + from connections: [AVCaptureConnection], + error: Error?) + { + guard let cont else { return } + self.cont = nil + + if let error { + let ns = error as NSError + if ns.domain == AVFoundationErrorDomain, + ns.code == AVError.maximumDurationReached.rawValue + { + cont.resume(returning: outputFileURL) + return + } + + self.logger.error("camera record failed: \(error.localizedDescription, privacy: .public)") + cont.resume(throwing: error) + return + } + + cont.resume(returning: outputFileURL) + } +} diff --git a/apps/macos/Sources/OpenClaw/CanvasA2UIActionMessageHandler.swift b/apps/macos/Sources/OpenClaw/CanvasA2UIActionMessageHandler.swift new file mode 100644 index 0000000000000000000000000000000000000000..2faca73c18f7a644d9226d33a8247813115f7e91 --- /dev/null +++ b/apps/macos/Sources/OpenClaw/CanvasA2UIActionMessageHandler.swift @@ -0,0 +1,149 @@ +import AppKit +import OpenClawIPC +import OpenClawKit +import Foundation +import WebKit + +final class CanvasA2UIActionMessageHandler: NSObject, WKScriptMessageHandler { + static let messageName = "openclawCanvasA2UIAction" + static let allMessageNames = [messageName] + + private let sessionKey: String + + init(sessionKey: String) { + self.sessionKey = sessionKey + super.init() + } + + func userContentController(_: WKUserContentController, didReceive message: WKScriptMessage) { + guard Self.allMessageNames.contains(message.name) else { return } + + // Only accept actions from local Canvas content (not arbitrary web pages). + guard let webView = message.webView, let url = webView.url else { return } + if let scheme = url.scheme, CanvasScheme.allSchemes.contains(scheme) { + // ok + } else if Self.isLocalNetworkCanvasURL(url) { + // ok + } else { + return + } + + let body: [String: Any] = { + if let dict = message.body as? [String: Any] { return dict } + if let dict = message.body as? [AnyHashable: Any] { + return dict.reduce(into: [String: Any]()) { acc, pair in + guard let key = pair.key as? String else { return } + acc[key] = pair.value + } + } + return [:] + }() + guard !body.isEmpty else { return } + + let userActionAny = body["userAction"] ?? body + let userAction: [String: Any] = { + if let dict = userActionAny as? [String: Any] { return dict } + if let dict = userActionAny as? [AnyHashable: Any] { + return dict.reduce(into: [String: Any]()) { acc, pair in + guard let key = pair.key as? String else { return } + acc[key] = pair.value + } + } + return [:] + }() + guard !userAction.isEmpty else { return } + + guard let name = OpenClawCanvasA2UIAction.extractActionName(userAction) else { return } + let actionId = + (userAction["id"] as? String)?.trimmingCharacters(in: .whitespacesAndNewlines).nonEmpty + ?? UUID().uuidString + + canvasWindowLogger.info("A2UI action \(name, privacy: .public) session=\(self.sessionKey, privacy: .public)") + + let surfaceId = (userAction["surfaceId"] as? String)?.trimmingCharacters(in: .whitespacesAndNewlines) + .nonEmpty ?? "main" + let sourceComponentId = (userAction["sourceComponentId"] as? String)? + .trimmingCharacters(in: .whitespacesAndNewlines).nonEmpty ?? "-" + let instanceId = InstanceIdentity.instanceId.lowercased() + let contextJSON = OpenClawCanvasA2UIAction.compactJSON(userAction["context"]) + + // Token-efficient and unambiguous. The agent should treat this as a UI event and (by default) update Canvas. + let messageContext = OpenClawCanvasA2UIAction.AgentMessageContext( + actionName: name, + session: .init(key: self.sessionKey, surfaceId: surfaceId), + component: .init(id: sourceComponentId, host: InstanceIdentity.displayName, instanceId: instanceId), + contextJSON: contextJSON) + let text = OpenClawCanvasA2UIAction.formatAgentMessage(messageContext) + + Task { [weak webView] in + if AppStateStore.shared.connectionMode == .local { + GatewayProcessManager.shared.setActive(true) + } + + let result = await GatewayConnection.shared.sendAgent( + GatewayAgentInvocation( + message: text, + sessionKey: self.sessionKey, + thinking: "low", + deliver: false, + to: nil, + channel: .last, + idempotencyKey: actionId)) + + await MainActor.run { + guard let webView else { return } + let js = OpenClawCanvasA2UIAction.jsDispatchA2UIActionStatus( + actionId: actionId, + ok: result.ok, + error: result.error) + webView.evaluateJavaScript(js) { _, _ in } + } + if !result.ok { + canvasWindowLogger.error( + """ + A2UI action send failed name=\(name, privacy: .public) \ + error=\(result.error ?? "unknown", privacy: .public) + """) + } + } + } + + static func isLocalNetworkCanvasURL(_ url: URL) -> Bool { + guard let scheme = url.scheme?.lowercased(), scheme == "http" || scheme == "https" else { + return false + } + guard let host = url.host?.trimmingCharacters(in: .whitespacesAndNewlines), !host.isEmpty else { + return false + } + if host == "localhost" { return true } + if host.hasSuffix(".local") { return true } + if host.hasSuffix(".ts.net") { return true } + if host.hasSuffix(".tailscale.net") { return true } + if !host.contains("."), !host.contains(":") { return true } + if let ipv4 = Self.parseIPv4(host) { + return Self.isLocalNetworkIPv4(ipv4) + } + return false + } + + static func parseIPv4(_ host: String) -> (UInt8, UInt8, UInt8, UInt8)? { + let parts = host.split(separator: ".", omittingEmptySubsequences: false) + guard parts.count == 4 else { return nil } + let bytes: [UInt8] = parts.compactMap { UInt8($0) } + guard bytes.count == 4 else { return nil } + return (bytes[0], bytes[1], bytes[2], bytes[3]) + } + + static func isLocalNetworkIPv4(_ ip: (UInt8, UInt8, UInt8, UInt8)) -> Bool { + let (a, b, _, _) = ip + if a == 10 { return true } + if a == 172, (16...31).contains(Int(b)) { return true } + if a == 192, b == 168 { return true } + if a == 127 { return true } + if a == 169, b == 254 { return true } + if a == 100, (64...127).contains(Int(b)) { return true } + return false + } + + // Formatting helpers live in OpenClawKit (`OpenClawCanvasA2UIAction`). +} diff --git a/apps/macos/Sources/OpenClaw/CanvasChromeContainerView.swift b/apps/macos/Sources/OpenClaw/CanvasChromeContainerView.swift new file mode 100644 index 0000000000000000000000000000000000000000..89c19ef1385641e62647e18868a801d50ad75729 --- /dev/null +++ b/apps/macos/Sources/OpenClaw/CanvasChromeContainerView.swift @@ -0,0 +1,225 @@ +import AppKit +import QuartzCore + +final class HoverChromeContainerView: NSView { + private let content: NSView + private let chrome: CanvasChromeOverlayView + private var tracking: NSTrackingArea? + var onClose: (() -> Void)? + + init(containing content: NSView) { + self.content = content + self.chrome = CanvasChromeOverlayView(frame: .zero) + super.init(frame: .zero) + + self.wantsLayer = true + self.layer?.cornerRadius = 12 + self.layer?.masksToBounds = true + self.layer?.backgroundColor = NSColor.windowBackgroundColor.cgColor + + self.content.translatesAutoresizingMaskIntoConstraints = false + self.addSubview(self.content) + + self.chrome.translatesAutoresizingMaskIntoConstraints = false + self.chrome.alphaValue = 0 + self.chrome.onClose = { [weak self] in self?.onClose?() } + self.addSubview(self.chrome) + + NSLayoutConstraint.activate([ + self.content.leadingAnchor.constraint(equalTo: self.leadingAnchor), + self.content.trailingAnchor.constraint(equalTo: self.trailingAnchor), + self.content.topAnchor.constraint(equalTo: self.topAnchor), + self.content.bottomAnchor.constraint(equalTo: self.bottomAnchor), + + self.chrome.leadingAnchor.constraint(equalTo: self.leadingAnchor), + self.chrome.trailingAnchor.constraint(equalTo: self.trailingAnchor), + self.chrome.topAnchor.constraint(equalTo: self.topAnchor), + self.chrome.bottomAnchor.constraint(equalTo: self.bottomAnchor), + ]) + } + + @available(*, unavailable) + required init?(coder: NSCoder) { fatalError("init(coder:) is not supported") } + + override func updateTrackingAreas() { + super.updateTrackingAreas() + if let tracking { + self.removeTrackingArea(tracking) + } + let area = NSTrackingArea( + rect: self.bounds, + options: [.activeAlways, .mouseEnteredAndExited, .inVisibleRect], + owner: self, + userInfo: nil) + self.addTrackingArea(area) + self.tracking = area + } + + private final class CanvasDragHandleView: NSView { + override func mouseDown(with event: NSEvent) { + self.window?.performDrag(with: event) + } + + override func acceptsFirstMouse(for _: NSEvent?) -> Bool { true } + } + + private final class CanvasResizeHandleView: NSView { + private var startPoint: NSPoint = .zero + private var startFrame: NSRect = .zero + + override func acceptsFirstMouse(for _: NSEvent?) -> Bool { true } + + override func mouseDown(with event: NSEvent) { + guard let window else { return } + _ = window.makeFirstResponder(self) + self.startPoint = NSEvent.mouseLocation + self.startFrame = window.frame + super.mouseDown(with: event) + } + + override func mouseDragged(with _: NSEvent) { + guard let window else { return } + let current = NSEvent.mouseLocation + let dx = current.x - self.startPoint.x + let dy = current.y - self.startPoint.y + + var frame = self.startFrame + frame.size.width = max(CanvasLayout.minPanelSize.width, frame.size.width + dx) + frame.origin.y += dy + frame.size.height = max(CanvasLayout.minPanelSize.height, frame.size.height - dy) + + if let screen = window.screen { + frame = CanvasWindowController.constrainFrame(frame, toVisibleFrame: screen.visibleFrame) + } + window.setFrame(frame, display: true) + } + } + + private final class CanvasChromeOverlayView: NSView { + var onClose: (() -> Void)? + + private let dragHandle = CanvasDragHandleView(frame: .zero) + private let resizeHandle = CanvasResizeHandleView(frame: .zero) + + private final class PassthroughVisualEffectView: NSVisualEffectView { + override func hitTest(_: NSPoint) -> NSView? { nil } + } + + private let closeBackground: NSVisualEffectView = { + let v = PassthroughVisualEffectView(frame: .zero) + v.material = .hudWindow + v.blendingMode = .withinWindow + v.state = .active + v.appearance = NSAppearance(named: .vibrantDark) + v.wantsLayer = true + v.layer?.cornerRadius = 10 + v.layer?.masksToBounds = true + v.layer?.borderWidth = 1 + v.layer?.borderColor = NSColor.white.withAlphaComponent(0.22).cgColor + v.layer?.backgroundColor = NSColor.black.withAlphaComponent(0.28).cgColor + v.layer?.shadowColor = NSColor.black.withAlphaComponent(0.35).cgColor + v.layer?.shadowOpacity = 0.35 + v.layer?.shadowRadius = 8 + v.layer?.shadowOffset = .zero + return v + }() + + private let closeButton: NSButton = { + let cfg = NSImage.SymbolConfiguration(pointSize: 8, weight: .semibold) + let img = NSImage(systemSymbolName: "xmark", accessibilityDescription: "Close")? + .withSymbolConfiguration(cfg) + ?? NSImage(size: NSSize(width: 18, height: 18)) + let btn = NSButton(image: img, target: nil, action: nil) + btn.isBordered = false + btn.bezelStyle = .regularSquare + btn.imageScaling = .scaleProportionallyDown + btn.contentTintColor = NSColor.white.withAlphaComponent(0.92) + btn.toolTip = "Close" + return btn + }() + + override init(frame frameRect: NSRect) { + super.init(frame: frameRect) + + self.wantsLayer = true + self.layer?.cornerRadius = 12 + self.layer?.masksToBounds = true + self.layer?.borderWidth = 1 + self.layer?.borderColor = NSColor.black.withAlphaComponent(0.18).cgColor + self.layer?.backgroundColor = NSColor.black.withAlphaComponent(0.02).cgColor + + self.dragHandle.translatesAutoresizingMaskIntoConstraints = false + self.dragHandle.wantsLayer = true + self.dragHandle.layer?.backgroundColor = NSColor.clear.cgColor + self.addSubview(self.dragHandle) + + self.resizeHandle.translatesAutoresizingMaskIntoConstraints = false + self.resizeHandle.wantsLayer = true + self.resizeHandle.layer?.backgroundColor = NSColor.clear.cgColor + self.addSubview(self.resizeHandle) + + self.closeBackground.translatesAutoresizingMaskIntoConstraints = false + self.addSubview(self.closeBackground) + + self.closeButton.translatesAutoresizingMaskIntoConstraints = false + self.closeButton.target = self + self.closeButton.action = #selector(self.handleClose) + self.addSubview(self.closeButton) + + NSLayoutConstraint.activate([ + self.dragHandle.leadingAnchor.constraint(equalTo: self.leadingAnchor), + self.dragHandle.trailingAnchor.constraint(equalTo: self.trailingAnchor), + self.dragHandle.topAnchor.constraint(equalTo: self.topAnchor), + self.dragHandle.heightAnchor.constraint(equalToConstant: 30), + + self.closeBackground.centerXAnchor.constraint(equalTo: self.closeButton.centerXAnchor), + self.closeBackground.centerYAnchor.constraint(equalTo: self.closeButton.centerYAnchor), + self.closeBackground.widthAnchor.constraint(equalToConstant: 20), + self.closeBackground.heightAnchor.constraint(equalToConstant: 20), + + self.closeButton.trailingAnchor.constraint(equalTo: self.trailingAnchor, constant: -8), + self.closeButton.topAnchor.constraint(equalTo: self.topAnchor, constant: 8), + self.closeButton.widthAnchor.constraint(equalToConstant: 16), + self.closeButton.heightAnchor.constraint(equalToConstant: 16), + + self.resizeHandle.trailingAnchor.constraint(equalTo: self.trailingAnchor), + self.resizeHandle.bottomAnchor.constraint(equalTo: self.bottomAnchor), + self.resizeHandle.widthAnchor.constraint(equalToConstant: 18), + self.resizeHandle.heightAnchor.constraint(equalToConstant: 18), + ]) + } + + @available(*, unavailable) + required init?(coder: NSCoder) { fatalError("init(coder:) is not supported") } + + override func hitTest(_ point: NSPoint) -> NSView? { + // When the chrome is hidden, do not intercept any mouse events (let the WKWebView receive them). + guard self.alphaValue > 0.02 else { return nil } + + if self.closeButton.frame.contains(point) { return self.closeButton } + if self.dragHandle.frame.contains(point) { return self.dragHandle } + if self.resizeHandle.frame.contains(point) { return self.resizeHandle } + return nil + } + + @objc private func handleClose() { + self.onClose?() + } + } + + override func mouseEntered(with _: NSEvent) { + NSAnimationContext.runAnimationGroup { ctx in + ctx.duration = 0.12 + ctx.timingFunction = CAMediaTimingFunction(name: .easeOut) + self.chrome.animator().alphaValue = 1 + } + } + + override func mouseExited(with _: NSEvent) { + NSAnimationContext.runAnimationGroup { ctx in + ctx.duration = 0.16 + ctx.timingFunction = CAMediaTimingFunction(name: .easeOut) + self.chrome.animator().alphaValue = 0 + } + } +} diff --git a/apps/macos/Sources/OpenClaw/CanvasFileWatcher.swift b/apps/macos/Sources/OpenClaw/CanvasFileWatcher.swift new file mode 100644 index 0000000000000000000000000000000000000000..3cf800fd10849ff2babba75daf04be4d3eff46a3 --- /dev/null +++ b/apps/macos/Sources/OpenClaw/CanvasFileWatcher.swift @@ -0,0 +1,94 @@ +import CoreServices +import Foundation + +final class CanvasFileWatcher: @unchecked Sendable { + private let url: URL + private let queue: DispatchQueue + private var stream: FSEventStreamRef? + private var pending = false + private let onChange: () -> Void + + init(url: URL, onChange: @escaping () -> Void) { + self.url = url + self.queue = DispatchQueue(label: "ai.openclaw.canvaswatcher") + self.onChange = onChange + } + + deinit { + self.stop() + } + + func start() { + guard self.stream == nil else { return } + + let retainedSelf = Unmanaged.passRetained(self) + var context = FSEventStreamContext( + version: 0, + info: retainedSelf.toOpaque(), + retain: nil, + release: { pointer in + guard let pointer else { return } + Unmanaged.fromOpaque(pointer).release() + }, + copyDescription: nil) + + let paths = [self.url.path] as CFArray + let flags = FSEventStreamCreateFlags( + kFSEventStreamCreateFlagFileEvents | + kFSEventStreamCreateFlagUseCFTypes | + kFSEventStreamCreateFlagNoDefer) + + guard let stream = FSEventStreamCreate( + kCFAllocatorDefault, + Self.callback, + &context, + paths, + FSEventStreamEventId(kFSEventStreamEventIdSinceNow), + 0.05, + flags) + else { + retainedSelf.release() + return + } + + self.stream = stream + FSEventStreamSetDispatchQueue(stream, self.queue) + if FSEventStreamStart(stream) == false { + self.stream = nil + FSEventStreamSetDispatchQueue(stream, nil) + FSEventStreamInvalidate(stream) + FSEventStreamRelease(stream) + } + } + + func stop() { + guard let stream = self.stream else { return } + self.stream = nil + FSEventStreamStop(stream) + FSEventStreamSetDispatchQueue(stream, nil) + FSEventStreamInvalidate(stream) + FSEventStreamRelease(stream) + } +} + +extension CanvasFileWatcher { + private static let callback: FSEventStreamCallback = { _, info, numEvents, _, eventFlags, _ in + guard let info else { return } + let watcher = Unmanaged.fromOpaque(info).takeUnretainedValue() + watcher.handleEvents(numEvents: numEvents, eventFlags: eventFlags) + } + + private func handleEvents(numEvents: Int, eventFlags: UnsafePointer?) { + guard numEvents > 0 else { return } + guard eventFlags != nil else { return } + + // Coalesce rapid changes (common during builds/atomic saves). + if self.pending { return } + self.pending = true + self.queue.asyncAfter(deadline: .now() + 0.12) { [weak self] in + guard let self else { return } + self.pending = false + self.onChange() + } + } +} diff --git a/apps/macos/Sources/OpenClaw/CanvasManager.swift b/apps/macos/Sources/OpenClaw/CanvasManager.swift new file mode 100644 index 0000000000000000000000000000000000000000..0055ffcfe210e936ec6c753eea07c1fcbe94ac31 --- /dev/null +++ b/apps/macos/Sources/OpenClaw/CanvasManager.swift @@ -0,0 +1,342 @@ +import AppKit +import OpenClawIPC +import OpenClawKit +import Foundation +import OSLog + +@MainActor +final class CanvasManager { + static let shared = CanvasManager() + + private static let logger = Logger(subsystem: "ai.openclaw", category: "CanvasManager") + + private var panelController: CanvasWindowController? + private var panelSessionKey: String? + private var lastAutoA2UIUrl: String? + private var gatewayWatchTask: Task? + + private init() { + self.startGatewayObserver() + } + + var onPanelVisibilityChanged: ((Bool) -> Void)? + + /// Optional anchor provider (e.g. menu bar status item). If nil, Canvas anchors to the mouse cursor. + var defaultAnchorProvider: (() -> NSRect?)? + + private nonisolated static let canvasRoot: URL = { + let base = FileManager().urls(for: .applicationSupportDirectory, in: .userDomainMask).first! + return base.appendingPathComponent("OpenClaw/canvas", isDirectory: true) + }() + + func show(sessionKey: String, path: String? = nil, placement: CanvasPlacement? = nil) throws -> String { + try self.showDetailed(sessionKey: sessionKey, target: path, placement: placement).directory + } + + func showDetailed( + sessionKey: String, + target: String? = nil, + placement: CanvasPlacement? = nil) throws -> CanvasShowResult + { + Self.logger.debug( + """ + showDetailed start session=\(sessionKey, privacy: .public) \ + target=\(target ?? "", privacy: .public) \ + placement=\(placement != nil) + """) + let anchorProvider = self.defaultAnchorProvider ?? Self.mouseAnchorProvider + let session = sessionKey.trimmingCharacters(in: .whitespacesAndNewlines) + let normalizedTarget = target? + .trimmingCharacters(in: .whitespacesAndNewlines) + .nonEmpty + + if let controller = self.panelController, self.panelSessionKey == session { + Self.logger.debug("showDetailed reuse existing session=\(session, privacy: .public)") + controller.onVisibilityChanged = { [weak self] visible in + self?.onPanelVisibilityChanged?(visible) + } + controller.presentAnchoredPanel(anchorProvider: anchorProvider) + controller.applyPreferredPlacement(placement) + self.refreshDebugStatus() + + // Existing session: only navigate when an explicit target was provided. + if let normalizedTarget { + controller.load(target: normalizedTarget) + return self.makeShowResult( + directory: controller.directoryPath, + target: target, + effectiveTarget: normalizedTarget) + } + + self.maybeAutoNavigateToA2UIAsync(controller: controller) + return CanvasShowResult( + directory: controller.directoryPath, + target: target, + effectiveTarget: nil, + status: .shown, + url: nil) + } + + Self.logger.debug("showDetailed creating new session=\(session, privacy: .public)") + self.panelController?.close() + self.panelController = nil + self.panelSessionKey = nil + + Self.logger.debug("showDetailed ensure canvas root dir") + try FileManager().createDirectory(at: Self.canvasRoot, withIntermediateDirectories: true) + Self.logger.debug("showDetailed init CanvasWindowController") + let controller = try CanvasWindowController( + sessionKey: session, + root: Self.canvasRoot, + presentation: .panel(anchorProvider: anchorProvider)) + Self.logger.debug("showDetailed CanvasWindowController init done") + controller.onVisibilityChanged = { [weak self] visible in + self?.onPanelVisibilityChanged?(visible) + } + self.panelController = controller + self.panelSessionKey = session + controller.applyPreferredPlacement(placement) + + // New session: default to "/" so the user sees either the welcome page or `index.html`. + let effectiveTarget = normalizedTarget ?? "/" + Self.logger.debug("showDetailed showCanvas effectiveTarget=\(effectiveTarget, privacy: .public)") + controller.showCanvas(path: effectiveTarget) + Self.logger.debug("showDetailed showCanvas done") + if normalizedTarget == nil { + self.maybeAutoNavigateToA2UIAsync(controller: controller) + } + self.refreshDebugStatus() + + return self.makeShowResult( + directory: controller.directoryPath, + target: target, + effectiveTarget: effectiveTarget) + } + + func hide(sessionKey: String) { + let session = sessionKey.trimmingCharacters(in: .whitespacesAndNewlines) + guard self.panelSessionKey == session else { return } + self.panelController?.hideCanvas() + } + + func hideAll() { + self.panelController?.hideCanvas() + } + + func eval(sessionKey: String, javaScript: String) async throws -> String { + _ = try self.show(sessionKey: sessionKey, path: nil) + guard let controller = self.panelController else { return "" } + return try await controller.eval(javaScript: javaScript) + } + + func snapshot(sessionKey: String, outPath: String?) async throws -> String { + _ = try self.show(sessionKey: sessionKey, path: nil) + guard let controller = self.panelController else { + throw NSError(domain: "Canvas", code: 21, userInfo: [NSLocalizedDescriptionKey: "canvas not available"]) + } + return try await controller.snapshot(to: outPath) + } + + // MARK: - Gateway A2UI auto-nav + + private func startGatewayObserver() { + self.gatewayWatchTask?.cancel() + self.gatewayWatchTask = Task { [weak self] in + guard let self else { return } + let stream = await GatewayConnection.shared.subscribe(bufferingNewest: 1) + for await push in stream { + self.handleGatewayPush(push) + } + } + } + + private func handleGatewayPush(_ push: GatewayPush) { + guard case let .snapshot(snapshot) = push else { return } + let raw = snapshot.canvashosturl?.trimmingCharacters(in: .whitespacesAndNewlines) ?? "" + if raw.isEmpty { + Self.logger.debug("canvas host url missing in gateway snapshot") + } else { + Self.logger.debug("canvas host url snapshot=\(raw, privacy: .public)") + } + let a2uiUrl = Self.resolveA2UIHostUrl(from: raw) + if a2uiUrl == nil, !raw.isEmpty { + Self.logger.debug("canvas host url invalid; cannot resolve A2UI") + } + guard let controller = self.panelController else { + if a2uiUrl != nil { + Self.logger.debug("canvas panel not visible; skipping auto-nav") + } + return + } + self.maybeAutoNavigateToA2UI(controller: controller, a2uiUrl: a2uiUrl) + } + + private func maybeAutoNavigateToA2UIAsync(controller: CanvasWindowController) { + Task { [weak self] in + guard let self else { return } + let a2uiUrl = await self.resolveA2UIHostUrl() + await MainActor.run { + guard self.panelController === controller else { return } + self.maybeAutoNavigateToA2UI(controller: controller, a2uiUrl: a2uiUrl) + } + } + } + + private func maybeAutoNavigateToA2UI(controller: CanvasWindowController, a2uiUrl: String?) { + guard let a2uiUrl else { return } + let shouldNavigate = controller.shouldAutoNavigateToA2UI(lastAutoTarget: self.lastAutoA2UIUrl) + guard shouldNavigate else { + Self.logger.debug("canvas auto-nav skipped; target unchanged") + return + } + Self.logger.debug("canvas auto-nav -> \(a2uiUrl, privacy: .public)") + controller.load(target: a2uiUrl) + self.lastAutoA2UIUrl = a2uiUrl + } + + private func resolveA2UIHostUrl() async -> String? { + let raw = await GatewayConnection.shared.canvasHostUrl() + return Self.resolveA2UIHostUrl(from: raw) + } + + func refreshDebugStatus() { + guard let controller = self.panelController else { return } + let enabled = AppStateStore.shared.debugPaneEnabled + let mode = AppStateStore.shared.connectionMode + let title: String? + let subtitle: String? + switch mode { + case .remote: + title = "Remote control" + switch ControlChannel.shared.state { + case .connected: + subtitle = "Connected" + case .connecting: + subtitle = "Connecting…" + case .disconnected: + subtitle = "Disconnected" + case let .degraded(message): + subtitle = message.isEmpty ? "Degraded" : message + } + case .local: + title = GatewayProcessManager.shared.status.label + subtitle = mode.rawValue + case .unconfigured: + title = "Unconfigured" + subtitle = mode.rawValue + } + controller.updateDebugStatus(enabled: enabled, title: title, subtitle: subtitle) + } + + private static func resolveA2UIHostUrl(from raw: String?) -> String? { + let trimmed = raw?.trimmingCharacters(in: .whitespacesAndNewlines) ?? "" + guard !trimmed.isEmpty, let base = URL(string: trimmed) else { return nil } + return base.appendingPathComponent("__openclaw__/a2ui/").absoluteString + "?platform=macos" + } + + // MARK: - Anchoring + + private static func mouseAnchorProvider() -> NSRect? { + let pt = NSEvent.mouseLocation + return NSRect(x: pt.x, y: pt.y, width: 1, height: 1) + } + + // placement interpretation is handled by the window controller. + + // MARK: - Helpers + + private static func directURL(for target: String?) -> URL? { + guard let target else { return nil } + let trimmed = target.trimmingCharacters(in: .whitespacesAndNewlines) + guard !trimmed.isEmpty else { return nil } + + if let url = URL(string: trimmed), let scheme = url.scheme?.lowercased() { + if scheme == "https" || scheme == "http" || scheme == "file" { return url } + } + + // Convenience: existing absolute *file* paths resolve as local files. + // (Avoid treating Canvas routes like "/" as filesystem paths.) + if trimmed.hasPrefix("/") { + var isDir: ObjCBool = false + if FileManager().fileExists(atPath: trimmed, isDirectory: &isDir), !isDir.boolValue { + return URL(fileURLWithPath: trimmed) + } + } + + return nil + } + + private func makeShowResult( + directory: String, + target: String?, + effectiveTarget: String) -> CanvasShowResult + { + if let url = Self.directURL(for: effectiveTarget) { + return CanvasShowResult( + directory: directory, + target: target, + effectiveTarget: effectiveTarget, + status: .web, + url: url.absoluteString) + } + + let sessionDir = URL(fileURLWithPath: directory) + let status = Self.localStatus(sessionDir: sessionDir, target: effectiveTarget) + let host = sessionDir.lastPathComponent + let canvasURL = CanvasScheme.makeURL(session: host, path: effectiveTarget)?.absoluteString + return CanvasShowResult( + directory: directory, + target: target, + effectiveTarget: effectiveTarget, + status: status, + url: canvasURL) + } + + private static func localStatus(sessionDir: URL, target: String) -> CanvasShowStatus { + let fm = FileManager() + let trimmed = target.trimmingCharacters(in: .whitespacesAndNewlines) + let withoutQuery = trimmed.split(separator: "?", maxSplits: 1, omittingEmptySubsequences: false).first + .map(String.init) ?? trimmed + var path = withoutQuery + if path.hasPrefix("/") { path.removeFirst() } + path = path.removingPercentEncoding ?? path + + // Root special-case: built-in scaffold page when no index exists. + if path.isEmpty { + let a = sessionDir.appendingPathComponent("index.html", isDirectory: false) + let b = sessionDir.appendingPathComponent("index.htm", isDirectory: false) + if fm.fileExists(atPath: a.path) || fm.fileExists(atPath: b.path) { return .ok } + return .welcome + } + + // Direct file or directory. + var candidate = sessionDir.appendingPathComponent(path, isDirectory: false) + var isDir: ObjCBool = false + if fm.fileExists(atPath: candidate.path, isDirectory: &isDir) { + if isDir.boolValue { + return Self.indexExists(in: candidate) ? .ok : .notFound + } + return .ok + } + + // Directory index behavior ("/yolo" -> "yolo/index.html") if directory exists. + if !path.isEmpty, !path.hasSuffix("/") { + candidate = sessionDir.appendingPathComponent(path, isDirectory: true) + if fm.fileExists(atPath: candidate.path, isDirectory: &isDir), isDir.boolValue { + return Self.indexExists(in: candidate) ? .ok : .notFound + } + } + + return .notFound + } + + private static func indexExists(in dir: URL) -> Bool { + let fm = FileManager() + let a = dir.appendingPathComponent("index.html", isDirectory: false) + if fm.fileExists(atPath: a.path) { return true } + let b = dir.appendingPathComponent("index.htm", isDirectory: false) + return fm.fileExists(atPath: b.path) + } + + // no bundled A2UI shell; scaffold fallback is purely visual +} diff --git a/apps/macos/Sources/OpenClaw/CanvasScheme.swift b/apps/macos/Sources/OpenClaw/CanvasScheme.swift new file mode 100644 index 0000000000000000000000000000000000000000..4f08da2d7b3003ae68b09c48c47b6d73101d0f94 --- /dev/null +++ b/apps/macos/Sources/OpenClaw/CanvasScheme.swift @@ -0,0 +1,42 @@ +import Foundation + +enum CanvasScheme { + static let scheme = "openclaw-canvas" + static let allSchemes = [scheme] + + static func makeURL(session: String, path: String? = nil) -> URL? { + var comps = URLComponents() + comps.scheme = Self.scheme + comps.host = session + let p = (path ?? "/").trimmingCharacters(in: .whitespacesAndNewlines) + if p.isEmpty || p == "/" { + comps.path = "/" + } else if p.hasPrefix("/") { + comps.path = p + } else { + comps.path = "/" + p + } + return comps.url + } + + static func mimeType(forExtension ext: String) -> String { + switch ext.lowercased() { + // Note: WKURLSchemeHandler uses URLResponse(mimeType:), which expects a bare MIME type + // (no `; charset=...`). Encoding is provided via URLResponse(textEncodingName:). + case "html", "htm": "text/html" + case "js", "mjs": "application/javascript" + case "css": "text/css" + case "json", "map": "application/json" + case "svg": "image/svg+xml" + case "png": "image/png" + case "jpg", "jpeg": "image/jpeg" + case "gif": "image/gif" + case "ico": "image/x-icon" + case "woff2": "font/woff2" + case "woff": "font/woff" + case "ttf": "font/ttf" + case "wasm": "application/wasm" + default: "application/octet-stream" + } + } +} diff --git a/apps/macos/Sources/OpenClaw/CanvasSchemeHandler.swift b/apps/macos/Sources/OpenClaw/CanvasSchemeHandler.swift new file mode 100644 index 0000000000000000000000000000000000000000..3241c08e0d2712a14e2854674dc7c75c4accdad9 --- /dev/null +++ b/apps/macos/Sources/OpenClaw/CanvasSchemeHandler.swift @@ -0,0 +1,259 @@ +import OpenClawKit +import Foundation +import OSLog +import WebKit + +private let canvasLogger = Logger(subsystem: "ai.openclaw", category: "Canvas") + +final class CanvasSchemeHandler: NSObject, WKURLSchemeHandler { + private let root: URL + + init(root: URL) { + self.root = root + } + + func webView(_: WKWebView, start urlSchemeTask: WKURLSchemeTask) { + guard let url = urlSchemeTask.request.url else { + urlSchemeTask.didFailWithError(NSError(domain: "Canvas", code: 1, userInfo: [ + NSLocalizedDescriptionKey: "missing url", + ])) + return + } + + let response = self.response(for: url) + let mime = response.mime + let data = response.data + let encoding = self.textEncodingName(forMimeType: mime) + + let urlResponse = URLResponse( + url: url, + mimeType: mime, + expectedContentLength: data.count, + textEncodingName: encoding) + urlSchemeTask.didReceive(urlResponse) + urlSchemeTask.didReceive(data) + urlSchemeTask.didFinish() + } + + func webView(_: WKWebView, stop _: WKURLSchemeTask) { + // no-op + } + + private struct CanvasResponse { + let mime: String + let data: Data + } + + private func response(for url: URL) -> CanvasResponse { + guard let scheme = url.scheme, CanvasScheme.allSchemes.contains(scheme) else { + return self.html("Invalid scheme.") + } + guard let session = url.host, !session.isEmpty else { + return self.html("Missing session.") + } + + // Keep session component safe; don't allow slashes or traversal. + if session.contains("/") || session.contains("..") { + return self.html("Invalid session.") + } + + let sessionRoot = self.root.appendingPathComponent(session, isDirectory: true) + + // Path mapping: request path maps directly into the session dir. + var path = url.path + if let qIdx = path.firstIndex(of: "?") { path = String(path[.. \(servedPath, privacy: .public)") + return CanvasResponse(mime: mime, data: data) + } catch { + let failedPath = standardizedFile.path + let errorText = error.localizedDescription + canvasLogger + .error( + "failed reading \(failedPath, privacy: .public): \(errorText, privacy: .public)") + return self.html("Failed to read file.", title: "Canvas error") + } + } + + private func resolveFileURL(sessionRoot: URL, requestPath: String) -> URL? { + let fm = FileManager() + var candidate = sessionRoot.appendingPathComponent(requestPath, isDirectory: false) + + var isDir: ObjCBool = false + if fm.fileExists(atPath: candidate.path, isDirectory: &isDir) { + if isDir.boolValue { + if let idx = self.resolveIndex(in: candidate) { return idx } + return nil + } + return candidate + } + + // Directory index behavior: + // - "/yolo" serves "/index.html" if that directory exists. + if !requestPath.isEmpty, !requestPath.hasSuffix("/") { + candidate = sessionRoot.appendingPathComponent(requestPath, isDirectory: true) + if fm.fileExists(atPath: candidate.path, isDirectory: &isDir), isDir.boolValue { + if let idx = self.resolveIndex(in: candidate) { return idx } + } + } + + // Root fallback: + // - "/" serves "/index.html" if present. + if requestPath.isEmpty { + return self.resolveIndex(in: sessionRoot) + } + + return nil + } + + private func resolveIndex(in dir: URL) -> URL? { + let fm = FileManager() + let a = dir.appendingPathComponent("index.html", isDirectory: false) + if fm.fileExists(atPath: a.path) { return a } + let b = dir.appendingPathComponent("index.htm", isDirectory: false) + if fm.fileExists(atPath: b.path) { return b } + return nil + } + + private func html(_ body: String, title: String = "Canvas") -> CanvasResponse { + let html = """ + + + + + + \(title) + + + +
+
\(body)
+
+ + + """ + return CanvasResponse(mime: "text/html", data: Data(html.utf8)) + } + + private func welcomePage(sessionRoot: URL) -> CanvasResponse { + let escaped = sessionRoot.path + .replacingOccurrences(of: "&", with: "&") + .replacingOccurrences(of: "<", with: "<") + .replacingOccurrences(of: ">", with: ">") + let body = """ +
Canvas is ready.
+
Create index.html in:
+
\(escaped)
+ """ + return self.html(body, title: "Canvas") + } + + private func scaffoldPage(sessionRoot: URL) -> CanvasResponse { + // Default Canvas UX: when no index exists, show the built-in scaffold page. + if let data = self.loadBundledResourceData(relativePath: "CanvasScaffold/scaffold.html") { + return CanvasResponse(mime: "text/html", data: data) + } + + // Fallback for dev misconfiguration: show the classic welcome page. + return self.welcomePage(sessionRoot: sessionRoot) + } + + private func loadBundledResourceData(relativePath: String) -> Data? { + let trimmed = relativePath.trimmingCharacters(in: .whitespacesAndNewlines) + guard !trimmed.isEmpty else { return nil } + if trimmed.contains("..") || trimmed.contains("\\") { return nil } + + let parts = trimmed.split(separator: "/") + guard let filename = parts.last else { return nil } + let subdirectory = + parts.count > 1 ? parts.dropLast().joined(separator: "/") : nil + let fileURL = URL(fileURLWithPath: String(filename)) + let ext = fileURL.pathExtension + let name = fileURL.deletingPathExtension().lastPathComponent + guard !name.isEmpty, !ext.isEmpty else { return nil } + + let bundle = OpenClawKitResources.bundle + let resourceURL = + bundle.url(forResource: name, withExtension: ext, subdirectory: subdirectory) + ?? bundle.url(forResource: name, withExtension: ext) + guard let resourceURL else { return nil } + return try? Data(contentsOf: resourceURL) + } + + private func textEncodingName(forMimeType mimeType: String) -> String? { + if mimeType.hasPrefix("text/") { return "utf-8" } + switch mimeType { + case "application/javascript", "application/json", "image/svg+xml": + return "utf-8" + default: + return nil + } + } +} + +#if DEBUG +extension CanvasSchemeHandler { + func _testResponse(for url: URL) -> (mime: String, data: Data) { + let response = self.response(for: url) + return (response.mime, response.data) + } + + func _testResolveFileURL(sessionRoot: URL, requestPath: String) -> URL? { + self.resolveFileURL(sessionRoot: sessionRoot, requestPath: requestPath) + } + + func _testTextEncodingName(for mimeType: String) -> String? { + self.textEncodingName(forMimeType: mimeType) + } +} +#endif diff --git a/apps/macos/Sources/OpenClaw/CanvasWindow.swift b/apps/macos/Sources/OpenClaw/CanvasWindow.swift new file mode 100644 index 0000000000000000000000000000000000000000..0cb3b7c0769af7870cb8028a4dee1ca18a668894 --- /dev/null +++ b/apps/macos/Sources/OpenClaw/CanvasWindow.swift @@ -0,0 +1,26 @@ +import AppKit + +let canvasWindowLogger = Logger(subsystem: "ai.openclaw", category: "Canvas") + +enum CanvasLayout { + static let panelSize = NSSize(width: 520, height: 680) + static let windowSize = NSSize(width: 1120, height: 840) + static let anchorPadding: CGFloat = 8 + static let defaultPadding: CGFloat = 10 + static let minPanelSize = NSSize(width: 360, height: 360) +} + +final class CanvasPanel: NSPanel { + override var canBecomeKey: Bool { true } + override var canBecomeMain: Bool { true } +} + +enum CanvasPresentation { + case window + case panel(anchorProvider: () -> NSRect?) + + var isPanel: Bool { + if case .panel = self { return true } + return false + } +} diff --git a/apps/macos/Sources/OpenClaw/CanvasWindowController+Helpers.swift b/apps/macos/Sources/OpenClaw/CanvasWindowController+Helpers.swift new file mode 100644 index 0000000000000000000000000000000000000000..a7d10f95b5623e25bc96e3fb855ede3c261c364c --- /dev/null +++ b/apps/macos/Sources/OpenClaw/CanvasWindowController+Helpers.swift @@ -0,0 +1,43 @@ +import AppKit +import Foundation + +extension CanvasWindowController { + // MARK: - Helpers + + static func sanitizeSessionKey(_ key: String) -> String { + let trimmed = key.trimmingCharacters(in: .whitespacesAndNewlines) + if trimmed.isEmpty { return "main" } + let allowed = CharacterSet(charactersIn: "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_-+") + let scalars = trimmed.unicodeScalars.map { allowed.contains($0) ? Character($0) : "_" } + return String(scalars) + } + + static func jsStringLiteral(_ value: String) -> String { + let data = try? JSONEncoder().encode(value) + return data.flatMap { String(data: $0, encoding: .utf8) } ?? "\"\"" + } + + static func jsOptionalStringLiteral(_ value: String?) -> String { + guard let value else { return "null" } + return Self.jsStringLiteral(value) + } + + static func storedFrameDefaultsKey(sessionKey: String) -> String { + "openclaw.canvas.frame.\(self.sanitizeSessionKey(sessionKey))" + } + + static func loadRestoredFrame(sessionKey: String) -> NSRect? { + let key = self.storedFrameDefaultsKey(sessionKey: sessionKey) + guard let arr = UserDefaults.standard.array(forKey: key) as? [Double], arr.count == 4 else { return nil } + let rect = NSRect(x: arr[0], y: arr[1], width: arr[2], height: arr[3]) + if rect.width < CanvasLayout.minPanelSize.width || rect.height < CanvasLayout.minPanelSize.height { return nil } + return rect + } + + static func storeRestoredFrame(_ frame: NSRect, sessionKey: String) { + let key = self.storedFrameDefaultsKey(sessionKey: sessionKey) + UserDefaults.standard.set( + [Double(frame.origin.x), Double(frame.origin.y), Double(frame.size.width), Double(frame.size.height)], + forKey: key) + } +} diff --git a/apps/macos/Sources/OpenClaw/CanvasWindowController+Navigation.swift b/apps/macos/Sources/OpenClaw/CanvasWindowController+Navigation.swift new file mode 100644 index 0000000000000000000000000000000000000000..7139b6834d4097c8164ca723fd92d200c9f156bd --- /dev/null +++ b/apps/macos/Sources/OpenClaw/CanvasWindowController+Navigation.swift @@ -0,0 +1,63 @@ +import AppKit +import WebKit + +extension CanvasWindowController { + // MARK: - WKNavigationDelegate + + @MainActor + func webView( + _: WKWebView, + decidePolicyFor navigationAction: WKNavigationAction, + decisionHandler: @escaping @MainActor @Sendable (WKNavigationActionPolicy) -> Void) + { + guard let url = navigationAction.request.url else { + decisionHandler(.cancel) + return + } + let scheme = url.scheme?.lowercased() + + // Deep links: allow local Canvas content to invoke the agent without bouncing through NSWorkspace. + if scheme == "openclaw" { + if let currentScheme = self.webView.url?.scheme, + CanvasScheme.allSchemes.contains(currentScheme) { + Task { await DeepLinkHandler.shared.handle(url: url) } + } else { + canvasWindowLogger + .debug("ignoring deep link from non-canvas page \(url.absoluteString, privacy: .public)") + } + decisionHandler(.cancel) + return + } + + // Keep web content inside the panel when reasonable. + // `about:blank` and friends are common internal navigations for WKWebView; never send them to NSWorkspace. + if CanvasScheme.allSchemes.contains(scheme ?? "") + || scheme == "https" + || scheme == "http" + || scheme == "about" + || scheme == "blob" + || scheme == "data" + || scheme == "javascript" + { + decisionHandler(.allow) + return + } + + // Only open external URLs when there is a registered handler, otherwise macOS will show a confusing + // "There is no application set to open the URL ..." alert (e.g. for about:blank). + if let appURL = NSWorkspace.shared.urlForApplication(toOpen: url) { + NSWorkspace.shared.open( + [url], + withApplicationAt: appURL, + configuration: NSWorkspace.OpenConfiguration(), + completionHandler: nil) + } else { + canvasWindowLogger.debug("no application to open url \(url.absoluteString, privacy: .public)") + } + decisionHandler(.cancel) + } + + func webView(_: WKWebView, didFinish _: WKNavigation?) { + self.applyDebugStatusIfNeeded() + } +} diff --git a/apps/macos/Sources/OpenClaw/CanvasWindowController+Testing.swift b/apps/macos/Sources/OpenClaw/CanvasWindowController+Testing.swift new file mode 100644 index 0000000000000000000000000000000000000000..6c53fbc9971cb73f2e25321a8a08931e8e07cd9b --- /dev/null +++ b/apps/macos/Sources/OpenClaw/CanvasWindowController+Testing.swift @@ -0,0 +1,39 @@ +#if DEBUG +import AppKit +import Foundation + +extension CanvasWindowController { + static func _testSanitizeSessionKey(_ key: String) -> String { + self.sanitizeSessionKey(key) + } + + static func _testJSStringLiteral(_ value: String) -> String { + self.jsStringLiteral(value) + } + + static func _testJSOptionalStringLiteral(_ value: String?) -> String { + self.jsOptionalStringLiteral(value) + } + + static func _testStoredFrameKey(sessionKey: String) -> String { + self.storedFrameDefaultsKey(sessionKey: sessionKey) + } + + static func _testStoreAndLoadFrame(sessionKey: String, frame: NSRect) -> NSRect? { + self.storeRestoredFrame(frame, sessionKey: sessionKey) + return self.loadRestoredFrame(sessionKey: sessionKey) + } + + static func _testParseIPv4(_ host: String) -> (UInt8, UInt8, UInt8, UInt8)? { + CanvasA2UIActionMessageHandler.parseIPv4(host) + } + + static func _testIsLocalNetworkIPv4(_ ip: (UInt8, UInt8, UInt8, UInt8)) -> Bool { + CanvasA2UIActionMessageHandler.isLocalNetworkIPv4(ip) + } + + static func _testIsLocalNetworkCanvasURL(_ url: URL) -> Bool { + CanvasA2UIActionMessageHandler.isLocalNetworkCanvasURL(url) + } +} +#endif diff --git a/apps/macos/Sources/OpenClaw/CanvasWindowController+Window.swift b/apps/macos/Sources/OpenClaw/CanvasWindowController+Window.swift new file mode 100644 index 0000000000000000000000000000000000000000..042ee00ba9781a9da0f0ff718152ee4fed00ebe4 --- /dev/null +++ b/apps/macos/Sources/OpenClaw/CanvasWindowController+Window.swift @@ -0,0 +1,166 @@ +import AppKit +import OpenClawIPC + +extension CanvasWindowController { + // MARK: - Window + + static func makeWindow(for presentation: CanvasPresentation, contentView: NSView) -> NSWindow { + switch presentation { + case .window: + let window = NSWindow( + contentRect: NSRect(origin: .zero, size: CanvasLayout.windowSize), + styleMask: [.titled, .closable, .resizable, .miniaturizable], + backing: .buffered, + defer: false) + window.title = "OpenClaw Canvas" + window.isReleasedWhenClosed = false + window.contentView = contentView + window.center() + window.minSize = NSSize(width: 880, height: 680) + return window + + case .panel: + let panel = CanvasPanel( + contentRect: NSRect(origin: .zero, size: CanvasLayout.panelSize), + styleMask: [.borderless, .resizable], + backing: .buffered, + defer: false) + // Keep Canvas below the Voice Wake overlay panel. + panel.level = NSWindow.Level(rawValue: NSWindow.Level.statusBar.rawValue - 1) + panel.hasShadow = true + panel.isMovable = false + panel.collectionBehavior = [.canJoinAllSpaces, .fullScreenAuxiliary] + panel.titleVisibility = .hidden + panel.titlebarAppearsTransparent = true + panel.backgroundColor = .clear + panel.isOpaque = false + panel.contentView = contentView + panel.becomesKeyOnlyIfNeeded = true + panel.hidesOnDeactivate = false + panel.minSize = CanvasLayout.minPanelSize + return panel + } + } + + func presentAnchoredPanel(anchorProvider: @escaping () -> NSRect?) { + guard case .panel = self.presentation, let window else { return } + self.repositionPanel(using: anchorProvider) + window.makeKeyAndOrderFront(nil) + NSApp.activate(ignoringOtherApps: true) + window.makeFirstResponder(self.webView) + VoiceWakeOverlayController.shared.bringToFrontIfVisible() + self.onVisibilityChanged?(true) + } + + func repositionPanel(using anchorProvider: () -> NSRect?) { + guard let panel = self.window else { return } + let anchor = anchorProvider() + let targetScreen = Self.screen(forAnchor: anchor) + ?? Self.screenContainingMouseCursor() + ?? panel.screen + ?? NSScreen.main + ?? NSScreen.screens.first + + let restored = Self.loadRestoredFrame(sessionKey: self.sessionKey) + let restoredIsValid = if let restored, let targetScreen { + Self.isFrameMeaningfullyVisible(restored, on: targetScreen) + } else { + restored != nil + } + + var frame = if let restored, restoredIsValid { + restored + } else { + Self.defaultTopRightFrame(panel: panel, screen: targetScreen) + } + + // Apply agent placement as partial overrides: + // - If agent provides x/y, override origin. + // - If agent provides width/height, override size. + // - If agent provides only size, keep the remembered origin. + if let placement = self.preferredPlacement { + if let x = placement.x { frame.origin.x = x } + if let y = placement.y { frame.origin.y = y } + if let w = placement.width { frame.size.width = max(CanvasLayout.minPanelSize.width, CGFloat(w)) } + if let h = placement.height { frame.size.height = max(CanvasLayout.minPanelSize.height, CGFloat(h)) } + } + + self.setPanelFrame(frame, on: targetScreen) + } + + static func defaultTopRightFrame(panel: NSWindow, screen: NSScreen?) -> NSRect { + let w = max(CanvasLayout.minPanelSize.width, panel.frame.width) + let h = max(CanvasLayout.minPanelSize.height, panel.frame.height) + return WindowPlacement.topRightFrame( + size: NSSize(width: w, height: h), + padding: CanvasLayout.defaultPadding, + on: screen) + } + + func setPanelFrame(_ frame: NSRect, on screen: NSScreen?) { + guard let panel = self.window else { return } + guard let s = screen ?? panel.screen ?? NSScreen.main ?? NSScreen.screens.first else { + panel.setFrame(frame, display: false) + self.persistFrameIfPanel() + return + } + + let constrained = Self.constrainFrame(frame, toVisibleFrame: s.visibleFrame) + panel.setFrame(constrained, display: false) + self.persistFrameIfPanel() + } + + static func screen(forAnchor anchor: NSRect?) -> NSScreen? { + guard let anchor else { return nil } + let center = NSPoint(x: anchor.midX, y: anchor.midY) + return NSScreen.screens.first { screen in + screen.frame.contains(anchor.origin) || screen.frame.contains(center) + } + } + + static func screenContainingMouseCursor() -> NSScreen? { + let point = NSEvent.mouseLocation + return NSScreen.screens.first { $0.frame.contains(point) } + } + + static func isFrameMeaningfullyVisible(_ frame: NSRect, on screen: NSScreen) -> Bool { + frame.intersects(screen.visibleFrame.insetBy(dx: 12, dy: 12)) + } + + static func constrainFrame(_ frame: NSRect, toVisibleFrame bounds: NSRect) -> NSRect { + if bounds == .zero { return frame } + + var next = frame + next.size.width = min(max(CanvasLayout.minPanelSize.width, next.size.width), bounds.width) + next.size.height = min(max(CanvasLayout.minPanelSize.height, next.size.height), bounds.height) + + let maxX = bounds.maxX - next.size.width + let maxY = bounds.maxY - next.size.height + + next.origin.x = maxX >= bounds.minX ? min(max(next.origin.x, bounds.minX), maxX) : bounds.minX + next.origin.y = maxY >= bounds.minY ? min(max(next.origin.y, bounds.minY), maxY) : bounds.minY + + next.origin.x = round(next.origin.x) + next.origin.y = round(next.origin.y) + return next + } + + // MARK: - NSWindowDelegate + + func windowWillClose(_: Notification) { + self.onVisibilityChanged?(false) + } + + func windowDidMove(_: Notification) { + self.persistFrameIfPanel() + } + + func windowDidEndLiveResize(_: Notification) { + self.persistFrameIfPanel() + } + + func persistFrameIfPanel() { + guard case .panel = self.presentation, let window else { return } + Self.storeRestoredFrame(window.frame, sessionKey: self.sessionKey) + } +} diff --git a/apps/macos/Sources/OpenClaw/CanvasWindowController.swift b/apps/macos/Sources/OpenClaw/CanvasWindowController.swift new file mode 100644 index 0000000000000000000000000000000000000000..ee15a6abb671bed0cb739c448b5c4c03b717e7dd --- /dev/null +++ b/apps/macos/Sources/OpenClaw/CanvasWindowController.swift @@ -0,0 +1,371 @@ +import AppKit +import OpenClawIPC +import OpenClawKit +import Foundation +import WebKit + +@MainActor +final class CanvasWindowController: NSWindowController, WKNavigationDelegate, NSWindowDelegate { + let sessionKey: String + private let root: URL + private let sessionDir: URL + private let schemeHandler: CanvasSchemeHandler + let webView: WKWebView + private var a2uiActionMessageHandler: CanvasA2UIActionMessageHandler? + private let watcher: CanvasFileWatcher + private let container: HoverChromeContainerView + let presentation: CanvasPresentation + var preferredPlacement: CanvasPlacement? + private(set) var currentTarget: String? + private var debugStatusEnabled = false + private var debugStatusTitle: String? + private var debugStatusSubtitle: String? + + var onVisibilityChanged: ((Bool) -> Void)? + + init(sessionKey: String, root: URL, presentation: CanvasPresentation) throws { + self.sessionKey = sessionKey + self.root = root + self.presentation = presentation + + canvasWindowLogger.debug("CanvasWindowController init start session=\(sessionKey, privacy: .public)") + let safeSessionKey = CanvasWindowController.sanitizeSessionKey(sessionKey) + canvasWindowLogger.debug("CanvasWindowController init sanitized session=\(safeSessionKey, privacy: .public)") + self.sessionDir = root.appendingPathComponent(safeSessionKey, isDirectory: true) + try FileManager().createDirectory(at: self.sessionDir, withIntermediateDirectories: true) + canvasWindowLogger.debug("CanvasWindowController init session dir ready") + + self.schemeHandler = CanvasSchemeHandler(root: root) + canvasWindowLogger.debug("CanvasWindowController init scheme handler ready") + + let config = WKWebViewConfiguration() + config.userContentController = WKUserContentController() + config.preferences.isElementFullscreenEnabled = true + config.preferences.setValue(true, forKey: "developerExtrasEnabled") + canvasWindowLogger.debug("CanvasWindowController init config ready") + for scheme in CanvasScheme.allSchemes { + config.setURLSchemeHandler(self.schemeHandler, forURLScheme: scheme) + } + canvasWindowLogger.debug("CanvasWindowController init scheme handler installed") + + // Bridge A2UI "a2uiaction" DOM events back into the native agent loop. + // + // Prefer WKScriptMessageHandler when WebKit exposes it, otherwise fall back to an unattended deep link + // (includes the app-generated key so it won't prompt). + canvasWindowLogger.debug("CanvasWindowController init building A2UI bridge script") + let deepLinkKey = DeepLinkHandler.currentCanvasKey() + let injectedSessionKey = sessionKey.trimmingCharacters(in: .whitespacesAndNewlines).nonEmpty ?? "main" + let bridgeScript = """ + (() => { + try { + const allowedSchemes = \(String(describing: CanvasScheme.allSchemes)); + const protocol = location.protocol.replace(':', ''); + if (!allowedSchemes.includes(protocol)) return; + if (globalThis.__openclawA2UIBridgeInstalled) return; + globalThis.__openclawA2UIBridgeInstalled = true; + + const deepLinkKey = \(Self.jsStringLiteral(deepLinkKey)); + const sessionKey = \(Self.jsStringLiteral(injectedSessionKey)); + const machineName = \(Self.jsStringLiteral(InstanceIdentity.displayName)); + const instanceId = \(Self.jsStringLiteral(InstanceIdentity.instanceId)); + + globalThis.addEventListener('a2uiaction', (evt) => { + try { + const payload = evt?.detail ?? evt?.payload ?? null; + if (!payload || payload.eventType !== 'a2ui.action') return; + + const action = payload.action ?? null; + const name = action?.name ?? ''; + if (!name) return; + + const context = Array.isArray(action?.context) ? action.context : []; + const userAction = { + id: (globalThis.crypto?.randomUUID?.() ?? String(Date.now())), + name, + surfaceId: payload.surfaceId ?? 'main', + sourceComponentId: payload.sourceComponentId ?? '', + dataContextPath: payload.dataContextPath ?? '', + timestamp: new Date().toISOString(), + ...(context.length ? { context } : {}), + }; + + const handler = globalThis.webkit?.messageHandlers?.openclawCanvasA2UIAction; + + // If the bundled A2UI shell is present, let it forward actions so we keep its richer + // context resolution (data model path lookups, surface detection, etc.). + const hasBundledA2UIHost = + !!globalThis.openclawA2UI || + !!document.querySelector('openclaw-a2ui-host'); + if (hasBundledA2UIHost && handler?.postMessage) return; + + // Otherwise, forward directly when possible. + if (!hasBundledA2UIHost && handler?.postMessage) { + handler.postMessage({ userAction }); + return; + } + + const ctx = userAction.context ? (' ctx=' + JSON.stringify(userAction.context)) : ''; + const message = + 'CANVAS_A2UI action=' + userAction.name + + ' session=' + sessionKey + + ' surface=' + userAction.surfaceId + + ' component=' + (userAction.sourceComponentId || '-') + + ' host=' + machineName.replace(/\\s+/g, '_') + + ' instance=' + instanceId + + ctx + + ' default=update_canvas'; + const params = new URLSearchParams(); + params.set('message', message); + params.set('sessionKey', sessionKey); + params.set('thinking', 'low'); + params.set('deliver', 'false'); + params.set('channel', 'last'); + params.set('key', deepLinkKey); + location.href = 'openclaw://agent?' + params.toString(); + } catch {} + }, true); + } catch {} + })(); + """ + config.userContentController.addUserScript( + WKUserScript(source: bridgeScript, injectionTime: .atDocumentStart, forMainFrameOnly: true)) + canvasWindowLogger.debug("CanvasWindowController init A2UI bridge installed") + + canvasWindowLogger.debug("CanvasWindowController init creating WKWebView") + self.webView = WKWebView(frame: .zero, configuration: config) + // Canvas scaffold is a fully self-contained HTML page; avoid relying on transparency underlays. + self.webView.setValue(true, forKey: "drawsBackground") + + let sessionDir = self.sessionDir + let webView = self.webView + self.watcher = CanvasFileWatcher(url: sessionDir) { [weak webView] in + Task { @MainActor in + guard let webView else { return } + + // Only auto-reload when we are showing local canvas content. + guard let scheme = webView.url?.scheme, + CanvasScheme.allSchemes.contains(scheme) else { return } + + let path = webView.url?.path ?? "" + if path == "/" || path.isEmpty { + let indexA = sessionDir.appendingPathComponent("index.html", isDirectory: false) + let indexB = sessionDir.appendingPathComponent("index.htm", isDirectory: false) + if !FileManager().fileExists(atPath: indexA.path), + !FileManager().fileExists(atPath: indexB.path) + { + return + } + } + + webView.reload() + } + } + + self.container = HoverChromeContainerView(containing: self.webView) + let window = Self.makeWindow(for: presentation, contentView: self.container) + canvasWindowLogger.debug("CanvasWindowController init makeWindow done") + super.init(window: window) + + let handler = CanvasA2UIActionMessageHandler(sessionKey: sessionKey) + self.a2uiActionMessageHandler = handler + for name in CanvasA2UIActionMessageHandler.allMessageNames { + self.webView.configuration.userContentController.add(handler, name: name) + } + + self.webView.navigationDelegate = self + self.window?.delegate = self + self.container.onClose = { [weak self] in + self?.hideCanvas() + } + + self.watcher.start() + canvasWindowLogger.debug("CanvasWindowController init done") + } + + @available(*, unavailable) + required init?(coder: NSCoder) { fatalError("init(coder:) is not supported") } + + @MainActor deinit { + for name in CanvasA2UIActionMessageHandler.allMessageNames { + self.webView.configuration.userContentController.removeScriptMessageHandler(forName: name) + } + self.watcher.stop() + } + + func applyPreferredPlacement(_ placement: CanvasPlacement?) { + self.preferredPlacement = placement + } + + func showCanvas(path: String? = nil) { + if case let .panel(anchorProvider) = self.presentation { + self.presentAnchoredPanel(anchorProvider: anchorProvider) + if let path { + self.load(target: path) + } + return + } + + self.showWindow(nil) + self.window?.makeKeyAndOrderFront(nil) + NSApp.activate(ignoringOtherApps: true) + if let path { + self.load(target: path) + } + self.onVisibilityChanged?(true) + } + + func hideCanvas() { + if case .panel = self.presentation { + self.persistFrameIfPanel() + } + self.window?.orderOut(nil) + self.onVisibilityChanged?(false) + } + + func load(target: String) { + let trimmed = target.trimmingCharacters(in: .whitespacesAndNewlines) + self.currentTarget = trimmed + + if let url = URL(string: trimmed), let scheme = url.scheme?.lowercased() { + if scheme == "https" || scheme == "http" { + canvasWindowLogger.debug("canvas load url \(url.absoluteString, privacy: .public)") + self.webView.load(URLRequest(url: url)) + return + } + if scheme == "file" { + canvasWindowLogger.debug("canvas load file \(url.absoluteString, privacy: .public)") + self.loadFile(url) + return + } + } + + // Convenience: absolute file paths resolve as local files when they exist. + // (Avoid treating Canvas routes like "/" as filesystem paths.) + if trimmed.hasPrefix("/") { + var isDir: ObjCBool = false + if FileManager().fileExists(atPath: trimmed, isDirectory: &isDir), !isDir.boolValue { + let url = URL(fileURLWithPath: trimmed) + canvasWindowLogger.debug("canvas load file \(url.absoluteString, privacy: .public)") + self.loadFile(url) + return + } + } + + guard let url = CanvasScheme.makeURL( + session: CanvasWindowController.sanitizeSessionKey(self.sessionKey), + path: trimmed) + else { + canvasWindowLogger + .error( + "invalid canvas url session=\(self.sessionKey, privacy: .public) path=\(trimmed, privacy: .public)") + return + } + canvasWindowLogger.debug("canvas load canvas \(url.absoluteString, privacy: .public)") + self.webView.load(URLRequest(url: url)) + } + + func updateDebugStatus(enabled: Bool, title: String?, subtitle: String?) { + self.debugStatusEnabled = enabled + self.debugStatusTitle = title + self.debugStatusSubtitle = subtitle + self.applyDebugStatusIfNeeded() + } + + func applyDebugStatusIfNeeded() { + let enabled = self.debugStatusEnabled + let title = Self.jsOptionalStringLiteral(self.debugStatusTitle) + let subtitle = Self.jsOptionalStringLiteral(self.debugStatusSubtitle) + let js = """ + (() => { + try { + const api = globalThis.__openclaw; + if (!api) return; + if (typeof api.setDebugStatusEnabled === 'function') { + api.setDebugStatusEnabled(\(enabled ? "true" : "false")); + } + if (!\(enabled ? "true" : "false")) return; + if (typeof api.setStatus === 'function') { + api.setStatus(\(title), \(subtitle)); + } + } catch (_) {} + })(); + """ + self.webView.evaluateJavaScript(js) { _, _ in } + } + + private func loadFile(_ url: URL) { + let fileURL = url.isFileURL ? url : URL(fileURLWithPath: url.path) + let accessDir = fileURL.deletingLastPathComponent() + self.webView.loadFileURL(fileURL, allowingReadAccessTo: accessDir) + } + + func eval(javaScript: String) async throws -> String { + try await withCheckedThrowingContinuation { cont in + self.webView.evaluateJavaScript(javaScript) { result, error in + if let error { + cont.resume(throwing: error) + return + } + if let result { + cont.resume(returning: String(describing: result)) + } else { + cont.resume(returning: "") + } + } + } + } + + func snapshot(to outPath: String?) async throws -> String { + let image: NSImage = try await withCheckedThrowingContinuation { cont in + self.webView.takeSnapshot(with: nil) { image, error in + if let error { + cont.resume(throwing: error) + return + } + guard let image else { + cont.resume(throwing: NSError(domain: "Canvas", code: 11, userInfo: [ + NSLocalizedDescriptionKey: "snapshot returned nil image", + ])) + return + } + cont.resume(returning: image) + } + } + + guard let tiff = image.tiffRepresentation, + let rep = NSBitmapImageRep(data: tiff), + let png = rep.representation(using: .png, properties: [:]) + else { + throw NSError(domain: "Canvas", code: 12, userInfo: [ + NSLocalizedDescriptionKey: "failed to encode png", + ]) + } + + let path: String + if let outPath, !outPath.isEmpty { + path = outPath + } else { + let ts = Int(Date().timeIntervalSince1970) + path = "/tmp/openclaw-canvas-\(CanvasWindowController.sanitizeSessionKey(self.sessionKey))-\(ts).png" + } + + try png.write(to: URL(fileURLWithPath: path), options: [.atomic]) + return path + } + + var directoryPath: String { + self.sessionDir.path + } + + func shouldAutoNavigateToA2UI(lastAutoTarget: String?) -> Bool { + let trimmed = (self.currentTarget ?? "").trimmingCharacters(in: .whitespacesAndNewlines) + if trimmed.isEmpty || trimmed == "/" { return true } + if let lastAuto = lastAutoTarget?.trimmingCharacters(in: .whitespacesAndNewlines), + !lastAuto.isEmpty, + trimmed == lastAuto + { + return true + } + return false + } +} diff --git a/apps/macos/Sources/OpenClaw/ChannelConfigForm.swift b/apps/macos/Sources/OpenClaw/ChannelConfigForm.swift new file mode 100644 index 0000000000000000000000000000000000000000..d00725be768e62bf721752097f3ddb2971696cba --- /dev/null +++ b/apps/macos/Sources/OpenClaw/ChannelConfigForm.swift @@ -0,0 +1,363 @@ +import SwiftUI + +struct ConfigSchemaForm: View { + @Bindable var store: ChannelsStore + let schema: ConfigSchemaNode + let path: ConfigPath + + var body: some View { + self.renderNode(self.schema, path: self.path) + } + + private func renderNode(_ schema: ConfigSchemaNode, path: ConfigPath) -> AnyView { + let storedValue = self.store.configValue(at: path) + let value = storedValue ?? schema.explicitDefault + let label = hintForPath(path, hints: store.configUiHints)?.label ?? schema.title + let help = hintForPath(path, hints: store.configUiHints)?.help ?? schema.description + let variants = schema.anyOf.isEmpty ? schema.oneOf : schema.anyOf + + if !variants.isEmpty { + let nonNull = variants.filter { !$0.isNullSchema } + if nonNull.count == 1, let only = nonNull.first { + return self.renderNode(only, path: path) + } + let literals = nonNull.compactMap(\.literalValue) + if !literals.isEmpty, literals.count == nonNull.count { + return AnyView( + VStack(alignment: .leading, spacing: 6) { + if let label { Text(label).font(.callout.weight(.semibold)) } + if let help { + Text(help) + .font(.caption) + .foregroundStyle(.secondary) + } + Picker( + "", + selection: self.enumBinding( + path, + options: literals, + defaultValue: schema.explicitDefault)) + { + Text("Select…").tag(-1) + ForEach(literals.indices, id: \ .self) { index in + Text(String(describing: literals[index])).tag(index) + } + } + .pickerStyle(.menu) + }) + } + } + + switch schema.schemaType { + case "object": + return AnyView( + VStack(alignment: .leading, spacing: 12) { + if let label { + Text(label) + .font(.callout.weight(.semibold)) + } + if let help { + Text(help) + .font(.caption) + .foregroundStyle(.secondary) + } + let properties = schema.properties + let sortedKeys = properties.keys.sorted { lhs, rhs in + let orderA = hintForPath(path + [.key(lhs)], hints: store.configUiHints)?.order ?? 0 + let orderB = hintForPath(path + [.key(rhs)], hints: store.configUiHints)?.order ?? 0 + if orderA != orderB { return orderA < orderB } + return lhs < rhs + } + ForEach(sortedKeys, id: \ .self) { key in + if let child = properties[key] { + self.renderNode(child, path: path + [.key(key)]) + } + } + if schema.allowsAdditionalProperties { + self.renderAdditionalProperties(schema, path: path, value: value) + } + }) + case "array": + return AnyView(self.renderArray(schema, path: path, value: value, label: label, help: help)) + case "boolean": + return AnyView( + Toggle(isOn: self.boolBinding(path, defaultValue: schema.explicitDefault as? Bool)) { + if let label { Text(label) } else { Text("Enabled") } + } + .help(help ?? "")) + case "number", "integer": + return AnyView(self.renderNumberField(schema, path: path, label: label, help: help)) + case "string": + return AnyView(self.renderStringField(schema, path: path, label: label, help: help)) + default: + return AnyView( + VStack(alignment: .leading, spacing: 6) { + if let label { Text(label).font(.callout.weight(.semibold)) } + Text("Unsupported field type.") + .font(.caption) + .foregroundStyle(.secondary) + }) + } + } + + @ViewBuilder + private func renderStringField( + _ schema: ConfigSchemaNode, + path: ConfigPath, + label: String?, + help: String?) -> some View + { + let hint = hintForPath(path, hints: store.configUiHints) + let placeholder = hint?.placeholder ?? "" + let sensitive = hint?.sensitive ?? isSensitivePath(path) + let defaultValue = schema.explicitDefault as? String + VStack(alignment: .leading, spacing: 6) { + if let label { Text(label).font(.callout.weight(.semibold)) } + if let help { + Text(help) + .font(.caption) + .foregroundStyle(.secondary) + } + if let options = schema.enumValues { + Picker("", selection: self.enumBinding(path, options: options, defaultValue: schema.explicitDefault)) { + Text("Select…").tag(-1) + ForEach(options.indices, id: \ .self) { index in + Text(String(describing: options[index])).tag(index) + } + } + .pickerStyle(.menu) + } else if sensitive { + SecureField(placeholder, text: self.stringBinding(path, defaultValue: defaultValue)) + .textFieldStyle(.roundedBorder) + } else { + TextField(placeholder, text: self.stringBinding(path, defaultValue: defaultValue)) + .textFieldStyle(.roundedBorder) + } + } + } + + @ViewBuilder + private func renderNumberField( + _ schema: ConfigSchemaNode, + path: ConfigPath, + label: String?, + help: String?) -> some View + { + let defaultValue = (schema.explicitDefault as? Double) + ?? (schema.explicitDefault as? Int).map(Double.init) + VStack(alignment: .leading, spacing: 6) { + if let label { Text(label).font(.callout.weight(.semibold)) } + if let help { + Text(help) + .font(.caption) + .foregroundStyle(.secondary) + } + TextField( + "", + text: self.numberBinding( + path, + isInteger: schema.schemaType == "integer", + defaultValue: defaultValue)) + .textFieldStyle(.roundedBorder) + } + } + + @ViewBuilder + private func renderArray( + _ schema: ConfigSchemaNode, + path: ConfigPath, + value: Any?, + label: String?, + help: String?) -> some View + { + let items = value as? [Any] ?? [] + let itemSchema = schema.items + VStack(alignment: .leading, spacing: 10) { + if let label { Text(label).font(.callout.weight(.semibold)) } + if let help { + Text(help) + .font(.caption) + .foregroundStyle(.secondary) + } + ForEach(items.indices, id: \ .self) { index in + HStack(alignment: .top, spacing: 8) { + if let itemSchema { + self.renderNode(itemSchema, path: path + [.index(index)]) + } else { + Text(String(describing: items[index])) + } + Button("Remove") { + var next = items + next.remove(at: index) + self.store.updateConfigValue(path: path, value: next) + } + .buttonStyle(.bordered) + .controlSize(.small) + } + } + Button("Add") { + var next = items + if let itemSchema { + next.append(itemSchema.defaultValue) + } else { + next.append("") + } + self.store.updateConfigValue(path: path, value: next) + } + .buttonStyle(.bordered) + .controlSize(.small) + } + } + + @ViewBuilder + private func renderAdditionalProperties( + _ schema: ConfigSchemaNode, + path: ConfigPath, + value: Any?) -> some View + { + if let additionalSchema = schema.additionalProperties { + let dict = value as? [String: Any] ?? [:] + let reserved = Set(schema.properties.keys) + let extras = dict.keys.filter { !reserved.contains($0) }.sorted() + + VStack(alignment: .leading, spacing: 8) { + Text("Extra entries") + .font(.callout.weight(.semibold)) + if extras.isEmpty { + Text("No extra entries yet.") + .font(.caption) + .foregroundStyle(.secondary) + } else { + ForEach(extras, id: \ .self) { key in + let itemPath: ConfigPath = path + [.key(key)] + HStack(alignment: .top, spacing: 8) { + TextField("Key", text: self.mapKeyBinding(path: path, key: key)) + .textFieldStyle(.roundedBorder) + .frame(width: 160) + self.renderNode(additionalSchema, path: itemPath) + Button("Remove") { + var next = dict + next.removeValue(forKey: key) + self.store.updateConfigValue(path: path, value: next) + } + .buttonStyle(.bordered) + .controlSize(.small) + } + } + } + Button("Add") { + var next = dict + var index = 1 + var key = "new-\(index)" + while next[key] != nil { + index += 1 + key = "new-\(index)" + } + next[key] = additionalSchema.defaultValue + self.store.updateConfigValue(path: path, value: next) + } + .buttonStyle(.bordered) + .controlSize(.small) + } + } + } + + private func stringBinding(_ path: ConfigPath, defaultValue: String?) -> Binding { + Binding( + get: { + if let value = store.configValue(at: path) as? String { return value } + return defaultValue ?? "" + }, + set: { newValue in + let trimmed = newValue.trimmingCharacters(in: .whitespacesAndNewlines) + self.store.updateConfigValue(path: path, value: trimmed.isEmpty ? nil : trimmed) + }) + } + + private func boolBinding(_ path: ConfigPath, defaultValue: Bool?) -> Binding { + Binding( + get: { + if let value = store.configValue(at: path) as? Bool { return value } + return defaultValue ?? false + }, + set: { newValue in + self.store.updateConfigValue(path: path, value: newValue) + }) + } + + private func numberBinding( + _ path: ConfigPath, + isInteger: Bool, + defaultValue: Double?) -> Binding + { + Binding( + get: { + if let value = store.configValue(at: path) { return String(describing: value) } + guard let defaultValue else { return "" } + return isInteger ? String(Int(defaultValue)) : String(defaultValue) + }, + set: { newValue in + let trimmed = newValue.trimmingCharacters(in: .whitespacesAndNewlines) + if trimmed.isEmpty { + self.store.updateConfigValue(path: path, value: nil) + } else if let value = Double(trimmed) { + self.store.updateConfigValue(path: path, value: isInteger ? Int(value) : value) + } + }) + } + + private func enumBinding( + _ path: ConfigPath, + options: [Any], + defaultValue: Any?) -> Binding + { + Binding( + get: { + let value = self.store.configValue(at: path) ?? defaultValue + guard let value else { return -1 } + return options.firstIndex { option in + String(describing: option) == String(describing: value) + } ?? -1 + }, + set: { index in + guard index >= 0, index < options.count else { + self.store.updateConfigValue(path: path, value: nil) + return + } + self.store.updateConfigValue(path: path, value: options[index]) + }) + } + + private func mapKeyBinding(path: ConfigPath, key: String) -> Binding { + Binding( + get: { key }, + set: { newValue in + let trimmed = newValue.trimmingCharacters(in: .whitespacesAndNewlines) + guard !trimmed.isEmpty else { return } + guard trimmed != key else { return } + let current = self.store.configValue(at: path) as? [String: Any] ?? [:] + guard current[trimmed] == nil else { return } + var next = current + next[trimmed] = current[key] + next.removeValue(forKey: key) + self.store.updateConfigValue(path: path, value: next) + }) + } +} + +struct ChannelConfigForm: View { + @Bindable var store: ChannelsStore + let channelId: String + + var body: some View { + if self.store.configSchemaLoading { + ProgressView().controlSize(.small) + } else if let schema = store.channelConfigSchema(for: channelId) { + ConfigSchemaForm(store: self.store, schema: schema, path: [.key("channels"), .key(self.channelId)]) + } else { + Text("Schema unavailable for this channel.") + .font(.caption) + .foregroundStyle(.secondary) + } + } +} diff --git a/apps/macos/Sources/OpenClaw/ChannelsSettings+ChannelSections.swift b/apps/macos/Sources/OpenClaw/ChannelsSettings+ChannelSections.swift new file mode 100644 index 0000000000000000000000000000000000000000..ea82aac013d32f7470b63b940b8319ae29b7c30c --- /dev/null +++ b/apps/macos/Sources/OpenClaw/ChannelsSettings+ChannelSections.swift @@ -0,0 +1,139 @@ +import SwiftUI + +extension ChannelsSettings { + func formSection(_ title: String, @ViewBuilder content: () -> some View) -> some View { + GroupBox(title) { + VStack(alignment: .leading, spacing: 10) { + content() + } + .frame(maxWidth: .infinity, alignment: .leading) + } + } + + @ViewBuilder + func channelHeaderActions(_ channel: ChannelItem) -> some View { + HStack(spacing: 8) { + if channel.id == "whatsapp" { + Button("Logout") { + Task { await self.store.logoutWhatsApp() } + } + .buttonStyle(.bordered) + .disabled(self.store.whatsappBusy) + } + + if channel.id == "telegram" { + Button("Logout") { + Task { await self.store.logoutTelegram() } + } + .buttonStyle(.bordered) + .disabled(self.store.telegramBusy) + } + + Button { + Task { await self.store.refresh(probe: true) } + } label: { + if self.store.isRefreshing { + ProgressView().controlSize(.small) + } else { + Text("Refresh") + } + } + .buttonStyle(.bordered) + .disabled(self.store.isRefreshing) + } + .controlSize(.small) + } + + var whatsAppSection: some View { + VStack(alignment: .leading, spacing: 16) { + self.formSection("Linking") { + if let message = self.store.whatsappLoginMessage { + Text(message) + .font(.caption) + .foregroundStyle(.secondary) + .fixedSize(horizontal: false, vertical: true) + } + + if let qr = self.store.whatsappLoginQrDataUrl, let image = self.qrImage(from: qr) { + Image(nsImage: image) + .resizable() + .interpolation(.none) + .frame(width: 180, height: 180) + .cornerRadius(8) + } + + HStack(spacing: 12) { + Button { + Task { await self.store.startWhatsAppLogin(force: false) } + } label: { + if self.store.whatsappBusy { + ProgressView().controlSize(.small) + } else { + Text("Show QR") + } + } + .buttonStyle(.borderedProminent) + .disabled(self.store.whatsappBusy) + + Button("Relink") { + Task { await self.store.startWhatsAppLogin(force: true) } + } + .buttonStyle(.bordered) + .disabled(self.store.whatsappBusy) + } + .font(.caption) + } + + self.configEditorSection(channelId: "whatsapp") + } + } + + @ViewBuilder + func genericChannelSection(_ channel: ChannelItem) -> some View { + VStack(alignment: .leading, spacing: 16) { + self.configEditorSection(channelId: channel.id) + } + } + + @ViewBuilder + private func configEditorSection(channelId: String) -> some View { + self.formSection("Configuration") { + ChannelConfigForm(store: self.store, channelId: channelId) + } + + self.configStatusMessage + + HStack(spacing: 12) { + Button { + Task { await self.store.saveConfigDraft() } + } label: { + if self.store.isSavingConfig { + ProgressView().controlSize(.small) + } else { + Text("Save") + } + } + .buttonStyle(.borderedProminent) + .disabled(self.store.isSavingConfig || !self.store.configDirty) + + Button("Reload") { + Task { await self.store.reloadConfigDraft() } + } + .buttonStyle(.bordered) + .disabled(self.store.isSavingConfig) + + Spacer() + } + .font(.caption) + } + + @ViewBuilder + var configStatusMessage: some View { + if let status = self.store.configStatus { + Text(status) + .font(.caption) + .foregroundStyle(.secondary) + .fixedSize(horizontal: false, vertical: true) + } + } +} diff --git a/apps/macos/Sources/OpenClaw/ChannelsSettings+ChannelState.swift b/apps/macos/Sources/OpenClaw/ChannelsSettings+ChannelState.swift new file mode 100644 index 0000000000000000000000000000000000000000..5be5818425b02b06c2977086d961964c516f7cfe --- /dev/null +++ b/apps/macos/Sources/OpenClaw/ChannelsSettings+ChannelState.swift @@ -0,0 +1,508 @@ +import OpenClawProtocol +import SwiftUI + +extension ChannelsSettings { + private func channelStatus( + _ id: String, + as type: T.Type) -> T? + { + self.store.snapshot?.decodeChannel(id, as: type) + } + + var whatsAppTint: Color { + guard let status = self.channelStatus("whatsapp", as: ChannelsStatusSnapshot.WhatsAppStatus.self) + else { return .secondary } + if !status.configured { return .secondary } + if !status.linked { return .red } + if status.lastError != nil { return .orange } + if status.connected { return .green } + if status.running { return .orange } + return .orange + } + + var telegramTint: Color { + guard let status = self.channelStatus("telegram", as: ChannelsStatusSnapshot.TelegramStatus.self) + else { return .secondary } + if !status.configured { return .secondary } + if status.lastError != nil { return .orange } + if status.probe?.ok == false { return .orange } + if status.running { return .green } + return .orange + } + + var discordTint: Color { + guard let status = self.channelStatus("discord", as: ChannelsStatusSnapshot.DiscordStatus.self) + else { return .secondary } + if !status.configured { return .secondary } + if status.lastError != nil { return .orange } + if status.probe?.ok == false { return .orange } + if status.running { return .green } + return .orange + } + + var googlechatTint: Color { + guard let status = self.channelStatus("googlechat", as: ChannelsStatusSnapshot.GoogleChatStatus.self) + else { return .secondary } + if !status.configured { return .secondary } + if status.lastError != nil { return .orange } + if status.probe?.ok == false { return .orange } + if status.running { return .green } + return .orange + } + + var signalTint: Color { + guard let status = self.channelStatus("signal", as: ChannelsStatusSnapshot.SignalStatus.self) + else { return .secondary } + if !status.configured { return .secondary } + if status.lastError != nil { return .orange } + if status.probe?.ok == false { return .orange } + if status.running { return .green } + return .orange + } + + var imessageTint: Color { + guard let status = self.channelStatus("imessage", as: ChannelsStatusSnapshot.IMessageStatus.self) + else { return .secondary } + if !status.configured { return .secondary } + if status.lastError != nil { return .orange } + if status.probe?.ok == false { return .orange } + if status.running { return .green } + return .orange + } + + var whatsAppSummary: String { + guard let status = self.channelStatus("whatsapp", as: ChannelsStatusSnapshot.WhatsAppStatus.self) + else { return "Checking…" } + if !status.linked { return "Not linked" } + if status.connected { return "Connected" } + if status.running { return "Running" } + return "Linked" + } + + var telegramSummary: String { + guard let status = self.channelStatus("telegram", as: ChannelsStatusSnapshot.TelegramStatus.self) + else { return "Checking…" } + if !status.configured { return "Not configured" } + if status.running { return "Running" } + return "Configured" + } + + var discordSummary: String { + guard let status = self.channelStatus("discord", as: ChannelsStatusSnapshot.DiscordStatus.self) + else { return "Checking…" } + if !status.configured { return "Not configured" } + if status.running { return "Running" } + return "Configured" + } + + var googlechatSummary: String { + guard let status = self.channelStatus("googlechat", as: ChannelsStatusSnapshot.GoogleChatStatus.self) + else { return "Checking…" } + if !status.configured { return "Not configured" } + if status.running { return "Running" } + return "Configured" + } + + var signalSummary: String { + guard let status = self.channelStatus("signal", as: ChannelsStatusSnapshot.SignalStatus.self) + else { return "Checking…" } + if !status.configured { return "Not configured" } + if status.running { return "Running" } + return "Configured" + } + + var imessageSummary: String { + guard let status = self.channelStatus("imessage", as: ChannelsStatusSnapshot.IMessageStatus.self) + else { return "Checking…" } + if !status.configured { return "Not configured" } + if status.running { return "Running" } + return "Configured" + } + + var whatsAppDetails: String? { + guard let status = self.channelStatus("whatsapp", as: ChannelsStatusSnapshot.WhatsAppStatus.self) + else { return nil } + var lines: [String] = [] + if let e164 = status.`self`?.e164 ?? status.`self`?.jid { + lines.append("Linked as \(e164)") + } + if let age = status.authAgeMs { + lines.append("Auth age \(msToAge(age))") + } + if let last = self.date(fromMs: status.lastConnectedAt) { + lines.append("Last connect \(relativeAge(from: last))") + } + if let disconnect = status.lastDisconnect { + let when = self.date(fromMs: disconnect.at).map { relativeAge(from: $0) } ?? "unknown" + let code = disconnect.status.map { "status \($0)" } ?? "status unknown" + let err = disconnect.error ?? "disconnect" + lines.append("Last disconnect \(code) · \(err) · \(when)") + } + if status.reconnectAttempts > 0 { + lines.append("Reconnect attempts \(status.reconnectAttempts)") + } + if let msgAt = self.date(fromMs: status.lastMessageAt) { + lines.append("Last message \(relativeAge(from: msgAt))") + } + if let err = status.lastError, !err.isEmpty { + lines.append("Error: \(err)") + } + return lines.isEmpty ? nil : lines.joined(separator: " · ") + } + + var telegramDetails: String? { + guard let status = self.channelStatus("telegram", as: ChannelsStatusSnapshot.TelegramStatus.self) + else { return nil } + var lines: [String] = [] + if let source = status.tokenSource { + lines.append("Token source: \(source)") + } + if let mode = status.mode { + lines.append("Mode: \(mode)") + } + if let probe = status.probe { + if probe.ok { + if let name = probe.bot?.username { + lines.append("Bot: @\(name)") + } + if let url = probe.webhook?.url, !url.isEmpty { + lines.append("Webhook: \(url)") + } + } else { + let code = probe.status.map { String($0) } ?? "unknown" + lines.append("Probe failed (\(code))") + } + } + if let last = self.date(fromMs: status.lastProbeAt) { + lines.append("Last probe \(relativeAge(from: last))") + } + if let err = status.lastError, !err.isEmpty { + lines.append("Error: \(err)") + } + return lines.isEmpty ? nil : lines.joined(separator: " · ") + } + + var discordDetails: String? { + guard let status = self.channelStatus("discord", as: ChannelsStatusSnapshot.DiscordStatus.self) + else { return nil } + var lines: [String] = [] + if let source = status.tokenSource { + lines.append("Token source: \(source)") + } + if let probe = status.probe { + if probe.ok { + if let name = probe.bot?.username { + lines.append("Bot: @\(name)") + } + if let elapsed = probe.elapsedMs { + lines.append("Probe \(Int(elapsed))ms") + } + } else { + let code = probe.status.map { String($0) } ?? "unknown" + lines.append("Probe failed (\(code))") + } + } + if let last = self.date(fromMs: status.lastProbeAt) { + lines.append("Last probe \(relativeAge(from: last))") + } + if let err = status.lastError, !err.isEmpty { + lines.append("Error: \(err)") + } + return lines.isEmpty ? nil : lines.joined(separator: " · ") + } + + var googlechatDetails: String? { + guard let status = self.channelStatus("googlechat", as: ChannelsStatusSnapshot.GoogleChatStatus.self) + else { return nil } + var lines: [String] = [] + if let source = status.credentialSource { + lines.append("Credential: \(source)") + } + if let audienceType = status.audienceType { + let audience = status.audience ?? "" + let label = audience.isEmpty ? audienceType : "\(audienceType) \(audience)" + lines.append("Audience: \(label)") + } + if let probe = status.probe { + if probe.ok { + if let elapsed = probe.elapsedMs { + lines.append("Probe \(Int(elapsed))ms") + } + } else { + let code = probe.status.map { String($0) } ?? "unknown" + lines.append("Probe failed (\(code))") + } + } + if let last = self.date(fromMs: status.lastProbeAt) { + lines.append("Last probe \(relativeAge(from: last))") + } + if let err = status.lastError, !err.isEmpty { + lines.append("Error: \(err)") + } + return lines.isEmpty ? nil : lines.joined(separator: " · ") + } + + var signalDetails: String? { + guard let status = self.channelStatus("signal", as: ChannelsStatusSnapshot.SignalStatus.self) + else { return nil } + var lines: [String] = [] + lines.append("Base URL: \(status.baseUrl)") + if let probe = status.probe { + if probe.ok { + if let version = probe.version, !version.isEmpty { + lines.append("Version \(version)") + } + if let elapsed = probe.elapsedMs { + lines.append("Probe \(Int(elapsed))ms") + } + } else { + let code = probe.status.map { String($0) } ?? "unknown" + lines.append("Probe failed (\(code))") + } + } + if let last = self.date(fromMs: status.lastProbeAt) { + lines.append("Last probe \(relativeAge(from: last))") + } + if let err = status.lastError, !err.isEmpty { + lines.append("Error: \(err)") + } + return lines.isEmpty ? nil : lines.joined(separator: " · ") + } + + var imessageDetails: String? { + guard let status = self.channelStatus("imessage", as: ChannelsStatusSnapshot.IMessageStatus.self) + else { return nil } + var lines: [String] = [] + if let cliPath = status.cliPath, !cliPath.isEmpty { + lines.append("CLI: \(cliPath)") + } + if let dbPath = status.dbPath, !dbPath.isEmpty { + lines.append("DB: \(dbPath)") + } + if let probe = status.probe, !probe.ok { + let err = probe.error ?? "probe failed" + lines.append("Probe error: \(err)") + } + if let last = self.date(fromMs: status.lastProbeAt) { + lines.append("Last probe \(relativeAge(from: last))") + } + if let err = status.lastError, !err.isEmpty { + lines.append("Error: \(err)") + } + return lines.isEmpty ? nil : lines.joined(separator: " · ") + } + + var orderedChannels: [ChannelItem] { + let fallback = ["whatsapp", "telegram", "discord", "googlechat", "slack", "signal", "imessage"] + let order = self.store.snapshot?.channelOrder ?? fallback + let channels = order.enumerated().map { index, id in + ChannelItem( + id: id, + title: self.resolveChannelTitle(id), + detailTitle: self.resolveChannelDetailTitle(id), + systemImage: self.resolveChannelSystemImage(id), + sortOrder: index) + } + return channels.sorted { lhs, rhs in + let lhsEnabled = self.channelEnabled(lhs) + let rhsEnabled = self.channelEnabled(rhs) + if lhsEnabled != rhsEnabled { return lhsEnabled && !rhsEnabled } + return lhs.sortOrder < rhs.sortOrder + } + } + + var enabledChannels: [ChannelItem] { + self.orderedChannels.filter { self.channelEnabled($0) } + } + + var availableChannels: [ChannelItem] { + self.orderedChannels.filter { !self.channelEnabled($0) } + } + + func ensureSelection() { + guard let selected = self.selectedChannel else { + self.selectedChannel = self.orderedChannels.first + return + } + if !self.orderedChannels.contains(selected) { + self.selectedChannel = self.orderedChannels.first + } + } + + func channelEnabled(_ channel: ChannelItem) -> Bool { + let status = self.channelStatusDictionary(channel.id) + let configured = status?["configured"]?.boolValue ?? false + let running = status?["running"]?.boolValue ?? false + let connected = status?["connected"]?.boolValue ?? false + let accountActive = self.store.snapshot?.channelAccounts[channel.id]?.contains( + where: { $0.configured == true || $0.running == true || $0.connected == true }) ?? false + return configured || running || connected || accountActive + } + + @ViewBuilder + func channelSection(_ channel: ChannelItem) -> some View { + if channel.id == "whatsapp" { + self.whatsAppSection + } else { + self.genericChannelSection(channel) + } + } + + func channelTint(_ channel: ChannelItem) -> Color { + switch channel.id { + case "whatsapp": + return self.whatsAppTint + case "telegram": + return self.telegramTint + case "discord": + return self.discordTint + case "googlechat": + return self.googlechatTint + case "signal": + return self.signalTint + case "imessage": + return self.imessageTint + default: + if self.channelHasError(channel) { return .orange } + if self.channelEnabled(channel) { return .green } + return .secondary + } + } + + func channelSummary(_ channel: ChannelItem) -> String { + switch channel.id { + case "whatsapp": + return self.whatsAppSummary + case "telegram": + return self.telegramSummary + case "discord": + return self.discordSummary + case "googlechat": + return self.googlechatSummary + case "signal": + return self.signalSummary + case "imessage": + return self.imessageSummary + default: + if self.channelHasError(channel) { return "Error" } + if self.channelEnabled(channel) { return "Active" } + return "Not configured" + } + } + + func channelDetails(_ channel: ChannelItem) -> String? { + switch channel.id { + case "whatsapp": + return self.whatsAppDetails + case "telegram": + return self.telegramDetails + case "discord": + return self.discordDetails + case "googlechat": + return self.googlechatDetails + case "signal": + return self.signalDetails + case "imessage": + return self.imessageDetails + default: + let status = self.channelStatusDictionary(channel.id) + if let err = status?["lastError"]?.stringValue, !err.isEmpty { + return "Error: \(err)" + } + return nil + } + } + + func channelLastCheckText(_ channel: ChannelItem) -> String { + guard let date = self.channelLastCheck(channel) else { return "never" } + return relativeAge(from: date) + } + + func channelLastCheck(_ channel: ChannelItem) -> Date? { + switch channel.id { + case "whatsapp": + guard let status = self.channelStatus("whatsapp", as: ChannelsStatusSnapshot.WhatsAppStatus.self) + else { return nil } + return self.date(fromMs: status.lastEventAt ?? status.lastMessageAt ?? status.lastConnectedAt) + case "telegram": + return self + .date(fromMs: self.channelStatus("telegram", as: ChannelsStatusSnapshot.TelegramStatus.self)? + .lastProbeAt) + case "discord": + return self + .date(fromMs: self.channelStatus("discord", as: ChannelsStatusSnapshot.DiscordStatus.self)? + .lastProbeAt) + case "googlechat": + return self + .date(fromMs: self.channelStatus("googlechat", as: ChannelsStatusSnapshot.GoogleChatStatus.self)? + .lastProbeAt) + case "signal": + return self + .date(fromMs: self.channelStatus("signal", as: ChannelsStatusSnapshot.SignalStatus.self)?.lastProbeAt) + case "imessage": + return self + .date(fromMs: self.channelStatus("imessage", as: ChannelsStatusSnapshot.IMessageStatus.self)? + .lastProbeAt) + default: + let status = self.channelStatusDictionary(channel.id) + if let probeAt = status?["lastProbeAt"]?.doubleValue { + return self.date(fromMs: probeAt) + } + if let accounts = self.store.snapshot?.channelAccounts[channel.id] { + let last = accounts.compactMap { $0.lastInboundAt ?? $0.lastOutboundAt }.max() + return self.date(fromMs: last) + } + return nil + } + } + + func channelHasError(_ channel: ChannelItem) -> Bool { + switch channel.id { + case "whatsapp": + guard let status = self.channelStatus("whatsapp", as: ChannelsStatusSnapshot.WhatsAppStatus.self) + else { return false } + return status.lastError?.isEmpty == false || status.lastDisconnect?.loggedOut == true + case "telegram": + guard let status = self.channelStatus("telegram", as: ChannelsStatusSnapshot.TelegramStatus.self) + else { return false } + return status.lastError?.isEmpty == false || status.probe?.ok == false + case "discord": + guard let status = self.channelStatus("discord", as: ChannelsStatusSnapshot.DiscordStatus.self) + else { return false } + return status.lastError?.isEmpty == false || status.probe?.ok == false + case "googlechat": + guard let status = self.channelStatus("googlechat", as: ChannelsStatusSnapshot.GoogleChatStatus.self) + else { return false } + return status.lastError?.isEmpty == false || status.probe?.ok == false + case "signal": + guard let status = self.channelStatus("signal", as: ChannelsStatusSnapshot.SignalStatus.self) + else { return false } + return status.lastError?.isEmpty == false || status.probe?.ok == false + case "imessage": + guard let status = self.channelStatus("imessage", as: ChannelsStatusSnapshot.IMessageStatus.self) + else { return false } + return status.lastError?.isEmpty == false || status.probe?.ok == false + default: + let status = self.channelStatusDictionary(channel.id) + return status?["lastError"]?.stringValue?.isEmpty == false + } + } + + private func resolveChannelTitle(_ id: String) -> String { + let label = self.store.resolveChannelLabel(id) + if label != id { return label } + return id.prefix(1).uppercased() + id.dropFirst() + } + + private func resolveChannelDetailTitle(_ id: String) -> String { + self.store.resolveChannelDetailLabel(id) + } + + private func resolveChannelSystemImage(_ id: String) -> String { + self.store.resolveChannelSystemImage(id) + } + + private func channelStatusDictionary(_ id: String) -> [String: AnyCodable]? { + self.store.snapshot?.channels[id]?.dictionaryValue + } +} diff --git a/apps/macos/Sources/OpenClaw/ChannelsSettings+Helpers.swift b/apps/macos/Sources/OpenClaw/ChannelsSettings+Helpers.swift new file mode 100644 index 0000000000000000000000000000000000000000..05b79ca049291f8f3dc7634ad2e64a861460db88 --- /dev/null +++ b/apps/macos/Sources/OpenClaw/ChannelsSettings+Helpers.swift @@ -0,0 +1,17 @@ +import AppKit + +extension ChannelsSettings { + func date(fromMs ms: Double?) -> Date? { + guard let ms else { return nil } + return Date(timeIntervalSince1970: ms / 1000) + } + + func qrImage(from dataUrl: String) -> NSImage? { + guard let comma = dataUrl.firstIndex(of: ",") else { return nil } + let header = dataUrl[.. some View { + ScrollView(.vertical) { + VStack(alignment: .leading, spacing: 16) { + self.detailHeader(for: channel) + Divider() + self.channelSection(channel) + Spacer(minLength: 0) + } + .frame(maxWidth: .infinity, alignment: .leading) + .padding(.horizontal, 24) + .padding(.vertical, 18) + } + } + + private func sidebarRow(_ channel: ChannelItem) -> some View { + let isSelected = self.selectedChannel == channel + return Button { + self.selectedChannel = channel + } label: { + HStack(spacing: 8) { + Circle() + .fill(self.channelTint(channel)) + .frame(width: 8, height: 8) + VStack(alignment: .leading, spacing: 2) { + Text(channel.title) + Text(self.channelSummary(channel)) + .font(.caption) + .foregroundStyle(.secondary) + } + } + .padding(.vertical, 4) + .padding(.horizontal, 6) + .frame(maxWidth: .infinity, alignment: .leading) + .background(isSelected ? Color.accentColor.opacity(0.18) : Color.clear) + .clipShape(RoundedRectangle(cornerRadius: 10, style: .continuous)) + .background(Color.clear) // ensure full-width hit test area + .contentShape(Rectangle()) + } + .frame(maxWidth: .infinity, alignment: .leading) + .buttonStyle(.plain) + .contentShape(Rectangle()) + } + + private func sidebarSectionHeader(_ title: String) -> some View { + Text(title) + .font(.caption.weight(.semibold)) + .foregroundStyle(.secondary) + .textCase(.uppercase) + .padding(.horizontal, 4) + .padding(.top, 2) + } + + private func detailHeader(for channel: ChannelItem) -> some View { + VStack(alignment: .leading, spacing: 8) { + HStack(alignment: .firstTextBaseline, spacing: 10) { + Label(channel.detailTitle, systemImage: channel.systemImage) + .font(.title3.weight(.semibold)) + self.statusBadge( + self.channelSummary(channel), + color: self.channelTint(channel)) + Spacer() + self.channelHeaderActions(channel) + } + + HStack(spacing: 10) { + Text("Last check \(self.channelLastCheckText(channel))") + .font(.caption) + .foregroundStyle(.secondary) + if self.channelHasError(channel) { + Text("Error") + .font(.caption2.weight(.semibold)) + .padding(.horizontal, 6) + .padding(.vertical, 2) + .background(Color.red.opacity(0.15)) + .foregroundStyle(.red) + .clipShape(Capsule()) + } + } + + if let details = self.channelDetails(channel) { + Text(details) + .font(.caption) + .foregroundStyle(.secondary) + .fixedSize(horizontal: false, vertical: true) + } + } + } + + private func statusBadge(_ text: String, color: Color) -> some View { + Text(text) + .font(.caption2.weight(.semibold)) + .padding(.horizontal, 8) + .padding(.vertical, 3) + .background(color.opacity(0.16)) + .foregroundStyle(color) + .clipShape(Capsule()) + } +} diff --git a/apps/macos/Sources/OpenClaw/ChannelsSettings.swift b/apps/macos/Sources/OpenClaw/ChannelsSettings.swift new file mode 100644 index 0000000000000000000000000000000000000000..b1177f0033bf3aa484d7961d02890c2d6cca089c --- /dev/null +++ b/apps/macos/Sources/OpenClaw/ChannelsSettings.swift @@ -0,0 +1,19 @@ +import AppKit +import SwiftUI + +struct ChannelsSettings: View { + struct ChannelItem: Identifiable, Hashable { + let id: String + let title: String + let detailTitle: String + let systemImage: String + let sortOrder: Int + } + + @Bindable var store: ChannelsStore + @State var selectedChannel: ChannelItem? + + init(store: ChannelsStore = .shared) { + self.store = store + } +} diff --git a/apps/macos/Sources/OpenClaw/ChannelsStore+Config.swift b/apps/macos/Sources/OpenClaw/ChannelsStore+Config.swift new file mode 100644 index 0000000000000000000000000000000000000000..c56cb320785481dbcee0a2184219fa4fc5635543 --- /dev/null +++ b/apps/macos/Sources/OpenClaw/ChannelsStore+Config.swift @@ -0,0 +1,154 @@ +import OpenClawProtocol +import Foundation + +extension ChannelsStore { + func loadConfigSchema() async { + guard !self.configSchemaLoading else { return } + self.configSchemaLoading = true + defer { self.configSchemaLoading = false } + + do { + let res: ConfigSchemaResponse = try await GatewayConnection.shared.requestDecoded( + method: .configSchema, + params: nil, + timeoutMs: 8000) + let schemaValue = res.schema.foundationValue + self.configSchema = ConfigSchemaNode(raw: schemaValue) + let hintValues = res.uihints.mapValues { $0.foundationValue } + self.configUiHints = decodeUiHints(hintValues) + } catch { + self.configStatus = error.localizedDescription + } + } + + func loadConfig() async { + do { + let snap: ConfigSnapshot = try await GatewayConnection.shared.requestDecoded( + method: .configGet, + params: nil, + timeoutMs: 10000) + self.configStatus = snap.valid == false + ? "Config invalid; fix it in ~/.openclaw/openclaw.json." + : nil + self.configRoot = snap.config?.mapValues { $0.foundationValue } ?? [:] + self.configDraft = cloneConfigValue(self.configRoot) as? [String: Any] ?? self.configRoot + self.configDirty = false + self.configLoaded = true + + self.applyUIConfig(snap) + } catch { + self.configStatus = error.localizedDescription + } + } + + private func applyUIConfig(_ snap: ConfigSnapshot) { + let ui = snap.config?["ui"]?.dictionaryValue + let rawSeam = ui?["seamColor"]?.stringValue?.trimmingCharacters(in: .whitespacesAndNewlines) ?? "" + AppStateStore.shared.seamColorHex = rawSeam.isEmpty ? nil : rawSeam + } + + func channelConfigSchema(for channelId: String) -> ConfigSchemaNode? { + guard let root = self.configSchema else { return nil } + return root.node(at: [.key("channels"), .key(channelId)]) + } + + func configValue(at path: ConfigPath) -> Any? { + if let value = valueAtPath(self.configDraft, path: path) { + return value + } + guard path.count >= 2 else { return nil } + if case .key("channels") = path[0], case .key = path[1] { + let fallbackPath = Array(path.dropFirst()) + return valueAtPath(self.configDraft, path: fallbackPath) + } + return nil + } + + func updateConfigValue(path: ConfigPath, value: Any?) { + var root: Any = self.configDraft + setValue(&root, path: path, value: value) + self.configDraft = root as? [String: Any] ?? self.configDraft + self.configDirty = true + } + + func saveConfigDraft() async { + guard !self.isSavingConfig else { return } + self.isSavingConfig = true + defer { self.isSavingConfig = false } + + do { + try await ConfigStore.save(self.configDraft) + await self.loadConfig() + } catch { + self.configStatus = error.localizedDescription + } + } + + func reloadConfigDraft() async { + await self.loadConfig() + } +} + +private func valueAtPath(_ root: Any, path: ConfigPath) -> Any? { + var current: Any? = root + for segment in path { + switch segment { + case let .key(key): + guard let dict = current as? [String: Any] else { return nil } + current = dict[key] + case let .index(index): + guard let array = current as? [Any], array.indices.contains(index) else { return nil } + current = array[index] + } + } + return current +} + +private func setValue(_ root: inout Any, path: ConfigPath, value: Any?) { + guard let segment = path.first else { return } + switch segment { + case let .key(key): + var dict = root as? [String: Any] ?? [:] + if path.count == 1 { + if let value { + dict[key] = value + } else { + dict.removeValue(forKey: key) + } + root = dict + return + } + var child = dict[key] ?? [:] + setValue(&child, path: Array(path.dropFirst()), value: value) + dict[key] = child + root = dict + case let .index(index): + var array = root as? [Any] ?? [] + if index >= array.count { + array.append(contentsOf: repeatElement(NSNull() as Any, count: index - array.count + 1)) + } + if path.count == 1 { + if let value { + array[index] = value + } else if array.indices.contains(index) { + array.remove(at: index) + } + root = array + return + } + var child = array[index] + setValue(&child, path: Array(path.dropFirst()), value: value) + array[index] = child + root = array + } +} + +private func cloneConfigValue(_ value: Any) -> Any { + guard JSONSerialization.isValidJSONObject(value) else { return value } + do { + let data = try JSONSerialization.data(withJSONObject: value, options: []) + return try JSONSerialization.jsonObject(with: data, options: []) + } catch { + return value + } +} diff --git a/apps/macos/Sources/OpenClaw/ChannelsStore+Lifecycle.swift b/apps/macos/Sources/OpenClaw/ChannelsStore+Lifecycle.swift new file mode 100644 index 0000000000000000000000000000000000000000..0610fe46438f39c7aefe1941db76fa10235a3f38 --- /dev/null +++ b/apps/macos/Sources/OpenClaw/ChannelsStore+Lifecycle.swift @@ -0,0 +1,163 @@ +import OpenClawProtocol +import Foundation + +extension ChannelsStore { + func start() { + guard !self.isPreview else { return } + guard self.pollTask == nil else { return } + self.pollTask = Task.detached { [weak self] in + guard let self else { return } + await self.refresh(probe: true) + await self.loadConfigSchema() + await self.loadConfig() + while !Task.isCancelled { + try? await Task.sleep(nanoseconds: UInt64(self.interval * 1_000_000_000)) + await self.refresh(probe: false) + } + } + } + + func stop() { + self.pollTask?.cancel() + self.pollTask = nil + } + + func refresh(probe: Bool) async { + guard !self.isRefreshing else { return } + self.isRefreshing = true + defer { self.isRefreshing = false } + + do { + let params: [String: AnyCodable] = [ + "probe": AnyCodable(probe), + "timeoutMs": AnyCodable(8000), + ] + let snap: ChannelsStatusSnapshot = try await GatewayConnection.shared.requestDecoded( + method: .channelsStatus, + params: params, + timeoutMs: 12000) + self.snapshot = snap + self.lastSuccess = Date() + self.lastError = nil + } catch { + self.lastError = error.localizedDescription + } + } + + func startWhatsAppLogin(force: Bool, autoWait: Bool = true) async { + guard !self.whatsappBusy else { return } + self.whatsappBusy = true + defer { self.whatsappBusy = false } + var shouldAutoWait = false + do { + let params: [String: AnyCodable] = [ + "force": AnyCodable(force), + "timeoutMs": AnyCodable(30000), + ] + let result: WhatsAppLoginStartResult = try await GatewayConnection.shared.requestDecoded( + method: .webLoginStart, + params: params, + timeoutMs: 35000) + self.whatsappLoginMessage = result.message + self.whatsappLoginQrDataUrl = result.qrDataUrl + self.whatsappLoginConnected = nil + shouldAutoWait = autoWait && result.qrDataUrl != nil + } catch { + self.whatsappLoginMessage = error.localizedDescription + self.whatsappLoginQrDataUrl = nil + self.whatsappLoginConnected = nil + } + await self.refresh(probe: true) + if shouldAutoWait { + Task { await self.waitWhatsAppLogin() } + } + } + + func waitWhatsAppLogin(timeoutMs: Int = 120_000) async { + guard !self.whatsappBusy else { return } + self.whatsappBusy = true + defer { self.whatsappBusy = false } + do { + let params: [String: AnyCodable] = [ + "timeoutMs": AnyCodable(timeoutMs), + ] + let result: WhatsAppLoginWaitResult = try await GatewayConnection.shared.requestDecoded( + method: .webLoginWait, + params: params, + timeoutMs: Double(timeoutMs) + 5000) + self.whatsappLoginMessage = result.message + self.whatsappLoginConnected = result.connected + if result.connected { + self.whatsappLoginQrDataUrl = nil + } + } catch { + self.whatsappLoginMessage = error.localizedDescription + } + await self.refresh(probe: true) + } + + func logoutWhatsApp() async { + guard !self.whatsappBusy else { return } + self.whatsappBusy = true + defer { self.whatsappBusy = false } + do { + let params: [String: AnyCodable] = [ + "channel": AnyCodable("whatsapp"), + ] + let result: ChannelLogoutResult = try await GatewayConnection.shared.requestDecoded( + method: .channelsLogout, + params: params, + timeoutMs: 15000) + self.whatsappLoginMessage = result.cleared + ? "Logged out and cleared credentials." + : "No WhatsApp session found." + self.whatsappLoginQrDataUrl = nil + } catch { + self.whatsappLoginMessage = error.localizedDescription + } + await self.refresh(probe: true) + } + + func logoutTelegram() async { + guard !self.telegramBusy else { return } + self.telegramBusy = true + defer { self.telegramBusy = false } + do { + let params: [String: AnyCodable] = [ + "channel": AnyCodable("telegram"), + ] + let result: ChannelLogoutResult = try await GatewayConnection.shared.requestDecoded( + method: .channelsLogout, + params: params, + timeoutMs: 15000) + if result.envToken == true { + self.configStatus = "Telegram token still set via env; config cleared." + } else { + self.configStatus = result.cleared + ? "Telegram token cleared." + : "No Telegram token configured." + } + await self.loadConfig() + } catch { + self.configStatus = error.localizedDescription + } + await self.refresh(probe: true) + } +} + +private struct WhatsAppLoginStartResult: Codable { + let qrDataUrl: String? + let message: String +} + +private struct WhatsAppLoginWaitResult: Codable { + let connected: Bool + let message: String +} + +private struct ChannelLogoutResult: Codable { + let channel: String? + let accountId: String? + let cleared: Bool + let envToken: Bool? +} diff --git a/apps/macos/Sources/OpenClaw/ChannelsStore.swift b/apps/macos/Sources/OpenClaw/ChannelsStore.swift new file mode 100644 index 0000000000000000000000000000000000000000..724862efd72dcfe2e56d6d518b42a6e36987b9eb --- /dev/null +++ b/apps/macos/Sources/OpenClaw/ChannelsStore.swift @@ -0,0 +1,296 @@ +import OpenClawProtocol +import Foundation +import Observation + +struct ChannelsStatusSnapshot: Codable { + struct WhatsAppSelf: Codable { + let e164: String? + let jid: String? + } + + struct WhatsAppDisconnect: Codable { + let at: Double + let status: Int? + let error: String? + let loggedOut: Bool? + } + + struct WhatsAppStatus: Codable { + let configured: Bool + let linked: Bool + let authAgeMs: Double? + let `self`: WhatsAppSelf? + let running: Bool + let connected: Bool + let lastConnectedAt: Double? + let lastDisconnect: WhatsAppDisconnect? + let reconnectAttempts: Int + let lastMessageAt: Double? + let lastEventAt: Double? + let lastError: String? + } + + struct TelegramBot: Codable { + let id: Int? + let username: String? + } + + struct TelegramWebhook: Codable { + let url: String? + let hasCustomCert: Bool? + } + + struct TelegramProbe: Codable { + let ok: Bool + let status: Int? + let error: String? + let elapsedMs: Double? + let bot: TelegramBot? + let webhook: TelegramWebhook? + } + + struct TelegramStatus: Codable { + let configured: Bool + let tokenSource: String? + let running: Bool + let mode: String? + let lastStartAt: Double? + let lastStopAt: Double? + let lastError: String? + let probe: TelegramProbe? + let lastProbeAt: Double? + } + + struct DiscordBot: Codable { + let id: String? + let username: String? + } + + struct DiscordProbe: Codable { + let ok: Bool + let status: Int? + let error: String? + let elapsedMs: Double? + let bot: DiscordBot? + } + + struct DiscordStatus: Codable { + let configured: Bool + let tokenSource: String? + let running: Bool + let lastStartAt: Double? + let lastStopAt: Double? + let lastError: String? + let probe: DiscordProbe? + let lastProbeAt: Double? + } + + struct GoogleChatProbe: Codable { + let ok: Bool + let status: Int? + let error: String? + let elapsedMs: Double? + } + + struct GoogleChatStatus: Codable { + let configured: Bool + let credentialSource: String? + let audienceType: String? + let audience: String? + let webhookPath: String? + let webhookUrl: String? + let running: Bool + let lastStartAt: Double? + let lastStopAt: Double? + let lastError: String? + let probe: GoogleChatProbe? + let lastProbeAt: Double? + } + + struct SignalProbe: Codable { + let ok: Bool + let status: Int? + let error: String? + let elapsedMs: Double? + let version: String? + } + + struct SignalStatus: Codable { + let configured: Bool + let baseUrl: String + let running: Bool + let lastStartAt: Double? + let lastStopAt: Double? + let lastError: String? + let probe: SignalProbe? + let lastProbeAt: Double? + } + + struct IMessageProbe: Codable { + let ok: Bool + let error: String? + } + + struct IMessageStatus: Codable { + let configured: Bool + let running: Bool + let lastStartAt: Double? + let lastStopAt: Double? + let lastError: String? + let cliPath: String? + let dbPath: String? + let probe: IMessageProbe? + let lastProbeAt: Double? + } + + struct ChannelAccountSnapshot: Codable { + let accountId: String + let name: String? + let enabled: Bool? + let configured: Bool? + let linked: Bool? + let running: Bool? + let connected: Bool? + let reconnectAttempts: Int? + let lastConnectedAt: Double? + let lastError: String? + let lastStartAt: Double? + let lastStopAt: Double? + let lastInboundAt: Double? + let lastOutboundAt: Double? + let lastProbeAt: Double? + let mode: String? + let dmPolicy: String? + let allowFrom: [String]? + let tokenSource: String? + let botTokenSource: String? + let appTokenSource: String? + let baseUrl: String? + let allowUnmentionedGroups: Bool? + let cliPath: String? + let dbPath: String? + let port: Int? + let probe: AnyCodable? + let audit: AnyCodable? + let application: AnyCodable? + } + + struct ChannelUiMetaEntry: Codable { + let id: String + let label: String + let detailLabel: String + let systemImage: String? + } + + let ts: Double + let channelOrder: [String] + let channelLabels: [String: String] + let channelDetailLabels: [String: String]? + let channelSystemImages: [String: String]? + let channelMeta: [ChannelUiMetaEntry]? + let channels: [String: AnyCodable] + let channelAccounts: [String: [ChannelAccountSnapshot]] + let channelDefaultAccountId: [String: String] + + func decodeChannel(_ id: String, as type: T.Type) -> T? { + guard let value = self.channels[id] else { return nil } + do { + let data = try JSONEncoder().encode(value) + return try JSONDecoder().decode(type, from: data) + } catch { + return nil + } + } +} + +struct ConfigSnapshot: Codable { + struct Issue: Codable { + let path: String + let message: String + } + + let path: String? + let exists: Bool? + let raw: String? + let hash: String? + let parsed: AnyCodable? + let valid: Bool? + let config: [String: AnyCodable]? + let issues: [Issue]? +} + +@MainActor +@Observable +final class ChannelsStore { + static let shared = ChannelsStore() + + var snapshot: ChannelsStatusSnapshot? + var lastError: String? + var lastSuccess: Date? + var isRefreshing = false + + var whatsappLoginMessage: String? + var whatsappLoginQrDataUrl: String? + var whatsappLoginConnected: Bool? + var whatsappBusy = false + var telegramBusy = false + + var configStatus: String? + var isSavingConfig = false + var configSchemaLoading = false + var configSchema: ConfigSchemaNode? + var configUiHints: [String: ConfigUiHint] = [:] + var configDraft: [String: Any] = [:] + var configDirty = false + + let interval: TimeInterval = 45 + let isPreview: Bool + var pollTask: Task? + var configRoot: [String: Any] = [:] + var configLoaded = false + + func channelMetaEntry(_ id: String) -> ChannelsStatusSnapshot.ChannelUiMetaEntry? { + self.snapshot?.channelMeta?.first(where: { $0.id == id }) + } + + func resolveChannelLabel(_ id: String) -> String { + if let meta = self.channelMetaEntry(id), !meta.label.isEmpty { + return meta.label + } + if let label = self.snapshot?.channelLabels[id], !label.isEmpty { + return label + } + return id + } + + func resolveChannelDetailLabel(_ id: String) -> String { + if let meta = self.channelMetaEntry(id), !meta.detailLabel.isEmpty { + return meta.detailLabel + } + if let detail = self.snapshot?.channelDetailLabels?[id], !detail.isEmpty { + return detail + } + return self.resolveChannelLabel(id) + } + + func resolveChannelSystemImage(_ id: String) -> String { + if let meta = self.channelMetaEntry(id), let symbol = meta.systemImage, !symbol.isEmpty { + return symbol + } + if let symbol = self.snapshot?.channelSystemImages?[id], !symbol.isEmpty { + return symbol + } + return "message" + } + + func orderedChannelIds() -> [String] { + if let meta = self.snapshot?.channelMeta, !meta.isEmpty { + return meta.map(\.id) + } + return self.snapshot?.channelOrder ?? [] + } + + init(isPreview: Bool = ProcessInfo.processInfo.isPreview) { + self.isPreview = isPreview + } +} diff --git a/apps/macos/Sources/OpenClaw/CommandResolver.swift b/apps/macos/Sources/OpenClaw/CommandResolver.swift new file mode 100644 index 0000000000000000000000000000000000000000..c17f64e30e7345cacc15dcbb3d85dfd6ebcf4cd8 --- /dev/null +++ b/apps/macos/Sources/OpenClaw/CommandResolver.swift @@ -0,0 +1,574 @@ +import Foundation + +enum CommandResolver { + private static let projectRootDefaultsKey = "openclaw.gatewayProjectRootPath" + private static let helperName = "openclaw" + + static func gatewayEntrypoint(in root: URL) -> String? { + let distEntry = root.appendingPathComponent("dist/index.js").path + if FileManager().isReadableFile(atPath: distEntry) { return distEntry } + let openclawEntry = root.appendingPathComponent("openclaw.mjs").path + if FileManager().isReadableFile(atPath: openclawEntry) { return openclawEntry } + let binEntry = root.appendingPathComponent("bin/openclaw.js").path + if FileManager().isReadableFile(atPath: binEntry) { return binEntry } + return nil + } + + static func runtimeResolution() -> Result { + RuntimeLocator.resolve(searchPaths: self.preferredPaths()) + } + + static func runtimeResolution(searchPaths: [String]?) -> Result { + RuntimeLocator.resolve(searchPaths: searchPaths ?? self.preferredPaths()) + } + + static func makeRuntimeCommand( + runtime: RuntimeResolution, + entrypoint: String, + subcommand: String, + extraArgs: [String]) -> [String] + { + [runtime.path, entrypoint, subcommand] + extraArgs + } + + static func runtimeErrorCommand(_ error: RuntimeResolutionError) -> [String] { + let message = RuntimeLocator.describeFailure(error) + return self.errorCommand(with: message) + } + + static func errorCommand(with message: String) -> [String] { + let script = """ + cat <<'__OPENCLAW_ERR__' >&2 + \(message) + __OPENCLAW_ERR__ + exit 1 + """ + return ["/bin/sh", "-c", script] + } + + static func projectRoot() -> URL { + if let stored = UserDefaults.standard.string(forKey: self.projectRootDefaultsKey), + let url = self.expandPath(stored), + FileManager().fileExists(atPath: url.path) + { + return url + } + let fallback = FileManager().homeDirectoryForCurrentUser + .appendingPathComponent("Projects/openclaw") + if FileManager().fileExists(atPath: fallback.path) { + return fallback + } + return FileManager().homeDirectoryForCurrentUser + } + + static func setProjectRoot(_ path: String) { + UserDefaults.standard.set(path, forKey: self.projectRootDefaultsKey) + } + + static func projectRootPath() -> String { + self.projectRoot().path + } + + static func preferredPaths() -> [String] { + let current = ProcessInfo.processInfo.environment["PATH"]? + .split(separator: ":").map(String.init) ?? [] + let home = FileManager().homeDirectoryForCurrentUser + let projectRoot = self.projectRoot() + return self.preferredPaths(home: home, current: current, projectRoot: projectRoot) + } + + static func preferredPaths(home: URL, current: [String], projectRoot: URL) -> [String] { + var extras = [ + home.appendingPathComponent("Library/pnpm").path, + "/opt/homebrew/bin", + "/usr/local/bin", + "/usr/bin", + "/bin", + ] + #if DEBUG + // Dev-only convenience. Avoid project-local PATH hijacking in release builds. + extras.insert(projectRoot.appendingPathComponent("node_modules/.bin").path, at: 0) + #endif + let openclawPaths = self.openclawManagedPaths(home: home) + if !openclawPaths.isEmpty { + extras.insert(contentsOf: openclawPaths, at: 1) + } + extras.insert(contentsOf: self.nodeManagerBinPaths(home: home), at: 1 + openclawPaths.count) + var seen = Set() + // Preserve order while stripping duplicates so PATH lookups remain deterministic. + return (extras + current).filter { seen.insert($0).inserted } + } + + private static func openclawManagedPaths(home: URL) -> [String] { + let bases = [ + home.appendingPathComponent(".openclaw"), + ] + var paths: [String] = [] + for base in bases { + let bin = base.appendingPathComponent("bin") + let nodeBin = base.appendingPathComponent("tools/node/bin") + if FileManager().fileExists(atPath: bin.path) { + paths.append(bin.path) + } + if FileManager().fileExists(atPath: nodeBin.path) { + paths.append(nodeBin.path) + } + } + return paths + } + + private static func nodeManagerBinPaths(home: URL) -> [String] { + var bins: [String] = [] + + // Volta + let volta = home.appendingPathComponent(".volta/bin") + if FileManager().fileExists(atPath: volta.path) { + bins.append(volta.path) + } + + // asdf + let asdf = home.appendingPathComponent(".asdf/shims") + if FileManager().fileExists(atPath: asdf.path) { + bins.append(asdf.path) + } + + // fnm + bins.append(contentsOf: self.versionedNodeBinPaths( + base: home.appendingPathComponent(".local/share/fnm/node-versions"), + suffix: "installation/bin")) + + // nvm + bins.append(contentsOf: self.versionedNodeBinPaths( + base: home.appendingPathComponent(".nvm/versions/node"), + suffix: "bin")) + + return bins + } + + private static func versionedNodeBinPaths(base: URL, suffix: String) -> [String] { + guard FileManager().fileExists(atPath: base.path) else { return [] } + let entries: [String] + do { + entries = try FileManager().contentsOfDirectory(atPath: base.path) + } catch { + return [] + } + + func parseVersion(_ name: String) -> [Int] { + let trimmed = name.hasPrefix("v") ? String(name.dropFirst()) : name + return trimmed.split(separator: ".").compactMap { Int($0) } + } + + let sorted = entries.sorted { a, b in + let va = parseVersion(a) + let vb = parseVersion(b) + let maxCount = max(va.count, vb.count) + for i in 0.. bi } + } + // If identical numerically, keep stable ordering. + return a > b + } + + var paths: [String] = [] + for entry in sorted { + let binDir = base.appendingPathComponent(entry).appendingPathComponent(suffix) + let node = binDir.appendingPathComponent("node") + if FileManager().isExecutableFile(atPath: node.path) { + paths.append(binDir.path) + } + } + return paths + } + + static func findExecutable(named name: String, searchPaths: [String]? = nil) -> String? { + for dir in searchPaths ?? self.preferredPaths() { + let candidate = (dir as NSString).appendingPathComponent(name) + if FileManager().isExecutableFile(atPath: candidate) { + return candidate + } + } + return nil + } + + static func openclawExecutable(searchPaths: [String]? = nil) -> String? { + self.findExecutable(named: self.helperName, searchPaths: searchPaths) + } + + static func projectOpenClawExecutable(projectRoot: URL? = nil) -> String? { + #if DEBUG + let root = projectRoot ?? self.projectRoot() + let candidate = root.appendingPathComponent("node_modules/.bin").appendingPathComponent(self.helperName).path + return FileManager().isExecutableFile(atPath: candidate) ? candidate : nil + #else + return nil + #endif + } + + static func nodeCliPath() -> String? { + let root = self.projectRoot() + let candidates = [ + root.appendingPathComponent("openclaw.mjs").path, + root.appendingPathComponent("bin/openclaw.js").path, + ] + for candidate in candidates where FileManager().isReadableFile(atPath: candidate) { + return candidate + } + return nil + } + + static func hasAnyOpenClawInvoker(searchPaths: [String]? = nil) -> Bool { + if self.openclawExecutable(searchPaths: searchPaths) != nil { return true } + if self.findExecutable(named: "pnpm", searchPaths: searchPaths) != nil { return true } + if self.findExecutable(named: "node", searchPaths: searchPaths) != nil, + self.nodeCliPath() != nil + { + return true + } + return false + } + + static func openclawNodeCommand( + subcommand: String, + extraArgs: [String] = [], + defaults: UserDefaults = .standard, + configRoot: [String: Any]? = nil, + searchPaths: [String]? = nil) -> [String] + { + let settings = self.connectionSettings(defaults: defaults, configRoot: configRoot) + if settings.mode == .remote, let ssh = self.sshNodeCommand( + subcommand: subcommand, + extraArgs: extraArgs, + settings: settings) + { + return ssh + } + + let runtimeResult = self.runtimeResolution(searchPaths: searchPaths) + + switch runtimeResult { + case let .success(runtime): + let root = self.projectRoot() + if let openclawPath = self.projectOpenClawExecutable(projectRoot: root) { + return [openclawPath, subcommand] + extraArgs + } + + if let entry = self.gatewayEntrypoint(in: root) { + return self.makeRuntimeCommand( + runtime: runtime, + entrypoint: entry, + subcommand: subcommand, + extraArgs: extraArgs) + } + if let pnpm = self.findExecutable(named: "pnpm", searchPaths: searchPaths) { + // Use --silent to avoid pnpm lifecycle banners that would corrupt JSON outputs. + return [pnpm, "--silent", "openclaw", subcommand] + extraArgs + } + if let openclawPath = self.openclawExecutable(searchPaths: searchPaths) { + return [openclawPath, subcommand] + extraArgs + } + + let missingEntry = """ + openclaw entrypoint missing (looked for dist/index.js or openclaw.mjs); run pnpm build. + """ + return self.errorCommand(with: missingEntry) + + case let .failure(error): + return self.runtimeErrorCommand(error) + } + } + + static func openclawCommand( + subcommand: String, + extraArgs: [String] = [], + defaults: UserDefaults = .standard, + configRoot: [String: Any]? = nil, + searchPaths: [String]? = nil) -> [String] + { + self.openclawNodeCommand( + subcommand: subcommand, + extraArgs: extraArgs, + defaults: defaults, + configRoot: configRoot, + searchPaths: searchPaths) + } + + // MARK: - SSH helpers + + private static func sshNodeCommand(subcommand: String, extraArgs: [String], settings: RemoteSettings) -> [String]? { + guard !settings.target.isEmpty else { return nil } + guard let parsed = self.parseSSHTarget(settings.target) else { return nil } + + // Run the real openclaw CLI on the remote host. + let exportedPath = [ + "/opt/homebrew/bin", + "/usr/local/bin", + "/usr/bin", + "/bin", + "/usr/sbin", + "/sbin", + "$HOME/Library/pnpm", + "$PATH", + ].joined(separator: ":") + let quotedArgs = ([subcommand] + extraArgs).map(self.shellQuote).joined(separator: " ") + let userPRJ = settings.projectRoot.trimmingCharacters(in: .whitespacesAndNewlines) + let userCLI = settings.cliPath.trimmingCharacters(in: .whitespacesAndNewlines) + + let projectSection = if userPRJ.isEmpty { + """ + DEFAULT_PRJ="$HOME/Projects/openclaw" + if [ -d "$DEFAULT_PRJ" ]; then + PRJ="$DEFAULT_PRJ" + cd "$PRJ" || { echo "Project root not found: $PRJ"; exit 127; } + fi + """ + } else { + """ + PRJ=\(self.shellQuote(userPRJ)) + cd "$PRJ" || { echo "Project root not found: $PRJ"; exit 127; } + """ + } + + let cliSection = if userCLI.isEmpty { + "" + } else { + """ + CLI_HINT=\(self.shellQuote(userCLI)) + if [ -n "$CLI_HINT" ]; then + if [ -x "$CLI_HINT" ]; then + CLI="$CLI_HINT" + "$CLI_HINT" \(quotedArgs); + exit $?; + elif [ -f "$CLI_HINT" ]; then + if command -v node >/dev/null 2>&1; then + CLI="node $CLI_HINT" + node "$CLI_HINT" \(quotedArgs); + exit $?; + fi + fi + fi + """ + } + + let scriptBody = """ + PATH=\(exportedPath); + CLI=""; + \(cliSection) + \(projectSection) + if command -v openclaw >/dev/null 2>&1; then + CLI="$(command -v openclaw)" + openclaw \(quotedArgs); + elif [ -n "${PRJ:-}" ] && [ -f "$PRJ/dist/index.js" ]; then + if command -v node >/dev/null 2>&1; then + CLI="node $PRJ/dist/index.js" + node "$PRJ/dist/index.js" \(quotedArgs); + else + echo "Node >=22 required on remote host"; exit 127; + fi + elif [ -n "${PRJ:-}" ] && [ -f "$PRJ/openclaw.mjs" ]; then + if command -v node >/dev/null 2>&1; then + CLI="node $PRJ/openclaw.mjs" + node "$PRJ/openclaw.mjs" \(quotedArgs); + else + echo "Node >=22 required on remote host"; exit 127; + fi + elif [ -n "${PRJ:-}" ] && [ -f "$PRJ/bin/openclaw.js" ]; then + if command -v node >/dev/null 2>&1; then + CLI="node $PRJ/bin/openclaw.js" + node "$PRJ/bin/openclaw.js" \(quotedArgs); + else + echo "Node >=22 required on remote host"; exit 127; + fi + elif command -v pnpm >/dev/null 2>&1; then + CLI="pnpm --silent openclaw" + pnpm --silent openclaw \(quotedArgs); + else + echo "openclaw CLI missing on remote host"; exit 127; + fi + """ + let options: [String] = [ + "-o", "BatchMode=yes", + "-o", "StrictHostKeyChecking=accept-new", + "-o", "UpdateHostKeys=yes", + ] + let args = self.sshArguments( + target: parsed, + identity: settings.identity, + options: options, + remoteCommand: ["/bin/sh", "-c", scriptBody]) + return ["/usr/bin/ssh"] + args + } + + struct RemoteSettings { + let mode: AppState.ConnectionMode + let target: String + let identity: String + let projectRoot: String + let cliPath: String + } + + static func connectionSettings( + defaults: UserDefaults = .standard, + configRoot: [String: Any]? = nil) -> RemoteSettings + { + let root = configRoot ?? OpenClawConfigFile.loadDict() + let mode = ConnectionModeResolver.resolve(root: root, defaults: defaults).mode + let target = defaults.string(forKey: remoteTargetKey) ?? "" + let identity = defaults.string(forKey: remoteIdentityKey) ?? "" + let projectRoot = defaults.string(forKey: remoteProjectRootKey) ?? "" + let cliPath = defaults.string(forKey: remoteCliPathKey) ?? "" + return RemoteSettings( + mode: mode, + target: self.sanitizedTarget(target), + identity: identity, + projectRoot: projectRoot, + cliPath: cliPath) + } + + static func connectionModeIsRemote(defaults: UserDefaults = .standard) -> Bool { + self.connectionSettings(defaults: defaults).mode == .remote + } + + private static func sanitizedTarget(_ raw: String) -> String { + let trimmed = raw.trimmingCharacters(in: .whitespacesAndNewlines) + if trimmed.hasPrefix("ssh ") { + return trimmed.replacingOccurrences(of: "ssh ", with: "").trimmingCharacters(in: .whitespacesAndNewlines) + } + return trimmed + } + + struct SSHParsedTarget { + let user: String? + let host: String + let port: Int + } + + static func parseSSHTarget(_ target: String) -> SSHParsedTarget? { + let trimmed = self.normalizeSSHTargetInput(target) + guard !trimmed.isEmpty else { return nil } + if trimmed.rangeOfCharacter(from: CharacterSet.whitespacesAndNewlines.union(.controlCharacters)) != nil { + return nil + } + let userHostPort: String + let user: String? + if let atRange = trimmed.range(of: "@") { + user = String(trimmed[.. 0, parsedPort <= 65535 else { + return nil + } + port = parsedPort + } else { + host = userHostPort + port = 22 + } + + return self.makeSSHTarget(user: user, host: host, port: port) + } + + static func sshTargetValidationMessage(_ target: String) -> String? { + let trimmed = self.normalizeSSHTargetInput(target) + guard !trimmed.isEmpty else { return nil } + if trimmed.hasPrefix("-") { + return "SSH target cannot start with '-'" + } + if trimmed.rangeOfCharacter(from: CharacterSet.whitespacesAndNewlines.union(.controlCharacters)) != nil { + return "SSH target cannot contain spaces" + } + if self.parseSSHTarget(trimmed) == nil { + return "SSH target must look like user@host[:port]" + } + return nil + } + + private static func shellQuote(_ text: String) -> String { + if text.isEmpty { return "''" } + let escaped = text.replacingOccurrences(of: "'", with: "'\\''") + return "'\(escaped)'" + } + + private static func expandPath(_ path: String) -> URL? { + var expanded = path + if expanded.hasPrefix("~") { + let home = FileManager().homeDirectoryForCurrentUser.path + expanded.replaceSubrange(expanded.startIndex...expanded.startIndex, with: home) + } + return URL(fileURLWithPath: expanded) + } + + private static func normalizeSSHTargetInput(_ target: String) -> String { + var trimmed = target.trimmingCharacters(in: .whitespacesAndNewlines) + if trimmed.hasPrefix("ssh ") { + trimmed = trimmed.replacingOccurrences(of: "ssh ", with: "") + .trimmingCharacters(in: .whitespacesAndNewlines) + } + return trimmed + } + + private static func isValidSSHComponent(_ value: String, allowLeadingDash: Bool = false) -> Bool { + if value.isEmpty { return false } + if !allowLeadingDash, value.hasPrefix("-") { return false } + let invalid = CharacterSet.whitespacesAndNewlines.union(.controlCharacters) + return value.rangeOfCharacter(from: invalid) == nil + } + + static func makeSSHTarget(user: String?, host: String, port: Int) -> SSHParsedTarget? { + let trimmedHost = host.trimmingCharacters(in: .whitespacesAndNewlines) + guard self.isValidSSHComponent(trimmedHost) else { return nil } + let trimmedUser = user?.trimmingCharacters(in: .whitespacesAndNewlines) + let normalizedUser: String? + if let trimmedUser { + guard self.isValidSSHComponent(trimmedUser) else { return nil } + normalizedUser = trimmedUser.isEmpty ? nil : trimmedUser + } else { + normalizedUser = nil + } + guard port > 0, port <= 65535 else { return nil } + return SSHParsedTarget(user: normalizedUser, host: trimmedHost, port: port) + } + + private static func sshTargetString(_ target: SSHParsedTarget) -> String { + target.user.map { "\($0)@\(target.host)" } ?? target.host + } + + static func sshArguments( + target: SSHParsedTarget, + identity: String, + options: [String], + remoteCommand: [String] = []) -> [String] + { + var args = options + if target.port > 0 { + args.append(contentsOf: ["-p", String(target.port)]) + } + let trimmedIdentity = identity.trimmingCharacters(in: .whitespacesAndNewlines) + if !trimmedIdentity.isEmpty { + // Only use IdentitiesOnly when an explicit identity file is provided. + // This allows 1Password SSH agent and other SSH agents to provide keys. + args.append(contentsOf: ["-o", "IdentitiesOnly=yes"]) + args.append(contentsOf: ["-i", trimmedIdentity]) + } + args.append("--") + args.append(self.sshTargetString(target)) + args.append(contentsOf: remoteCommand) + return args + } + + #if SWIFT_PACKAGE + static func _testNodeManagerBinPaths(home: URL) -> [String] { + self.nodeManagerBinPaths(home: home) + } + #endif +} diff --git a/apps/macos/Sources/OpenClaw/ConfigFileWatcher.swift b/apps/macos/Sources/OpenClaw/ConfigFileWatcher.swift new file mode 100644 index 0000000000000000000000000000000000000000..23689f1fb9d904914afaff1bc25ead0d9e49338a --- /dev/null +++ b/apps/macos/Sources/OpenClaw/ConfigFileWatcher.swift @@ -0,0 +1,118 @@ +import CoreServices +import Foundation + +final class ConfigFileWatcher: @unchecked Sendable { + private let url: URL + private let queue: DispatchQueue + private var stream: FSEventStreamRef? + private var pending = false + private let onChange: () -> Void + private let watchedDir: URL + private let targetPath: String + private let targetName: String + + init(url: URL, onChange: @escaping () -> Void) { + self.url = url + self.queue = DispatchQueue(label: "ai.openclaw.configwatcher") + self.onChange = onChange + self.watchedDir = url.deletingLastPathComponent() + self.targetPath = url.path + self.targetName = url.lastPathComponent + } + + deinit { + self.stop() + } + + func start() { + guard self.stream == nil else { return } + + let retainedSelf = Unmanaged.passRetained(self) + var context = FSEventStreamContext( + version: 0, + info: retainedSelf.toOpaque(), + retain: nil, + release: { pointer in + guard let pointer else { return } + Unmanaged.fromOpaque(pointer).release() + }, + copyDescription: nil) + + let paths = [self.watchedDir.path] as CFArray + let flags = FSEventStreamCreateFlags( + kFSEventStreamCreateFlagFileEvents | + kFSEventStreamCreateFlagUseCFTypes | + kFSEventStreamCreateFlagNoDefer) + + guard let stream = FSEventStreamCreate( + kCFAllocatorDefault, + Self.callback, + &context, + paths, + FSEventStreamEventId(kFSEventStreamEventIdSinceNow), + 0.05, + flags) + else { + retainedSelf.release() + return + } + + self.stream = stream + FSEventStreamSetDispatchQueue(stream, self.queue) + if FSEventStreamStart(stream) == false { + self.stream = nil + FSEventStreamSetDispatchQueue(stream, nil) + FSEventStreamInvalidate(stream) + FSEventStreamRelease(stream) + } + } + + func stop() { + guard let stream = self.stream else { return } + self.stream = nil + FSEventStreamStop(stream) + FSEventStreamSetDispatchQueue(stream, nil) + FSEventStreamInvalidate(stream) + FSEventStreamRelease(stream) + } +} + +extension ConfigFileWatcher { + private static let callback: FSEventStreamCallback = { _, info, numEvents, eventPaths, eventFlags, _ in + guard let info else { return } + let watcher = Unmanaged.fromOpaque(info).takeUnretainedValue() + watcher.handleEvents( + numEvents: numEvents, + eventPaths: eventPaths, + eventFlags: eventFlags) + } + + private func handleEvents( + numEvents: Int, + eventPaths: UnsafeMutableRawPointer?, + eventFlags: UnsafePointer?) + { + guard numEvents > 0 else { return } + guard eventFlags != nil else { return } + guard self.matchesTarget(eventPaths: eventPaths) else { return } + + if self.pending { return } + self.pending = true + self.queue.asyncAfter(deadline: .now() + 0.12) { [weak self] in + guard let self else { return } + self.pending = false + self.onChange() + } + } + + private func matchesTarget(eventPaths: UnsafeMutableRawPointer?) -> Bool { + guard let eventPaths else { return true } + let paths = unsafeBitCast(eventPaths, to: NSArray.self) + for case let path as String in paths { + if path == self.targetPath { return true } + if path.hasSuffix("/\(self.targetName)") { return true } + if path == self.watchedDir.path { return true } + } + return false + } +} diff --git a/apps/macos/Sources/OpenClaw/ConfigSchemaSupport.swift b/apps/macos/Sources/OpenClaw/ConfigSchemaSupport.swift new file mode 100644 index 0000000000000000000000000000000000000000..4a7d4e0a48af1e85c418f3bd2f2bef5b9263757e --- /dev/null +++ b/apps/macos/Sources/OpenClaw/ConfigSchemaSupport.swift @@ -0,0 +1,204 @@ +import Foundation + +enum ConfigPathSegment: Hashable { + case key(String) + case index(Int) +} + +typealias ConfigPath = [ConfigPathSegment] + +struct ConfigUiHint { + let label: String? + let help: String? + let order: Double? + let advanced: Bool? + let sensitive: Bool? + let placeholder: String? + + init(raw: [String: Any]) { + self.label = raw["label"] as? String + self.help = raw["help"] as? String + if let order = raw["order"] as? Double { + self.order = order + } else if let orderInt = raw["order"] as? Int { + self.order = Double(orderInt) + } else { + self.order = nil + } + self.advanced = raw["advanced"] as? Bool + self.sensitive = raw["sensitive"] as? Bool + self.placeholder = raw["placeholder"] as? String + } +} + +struct ConfigSchemaNode { + let raw: [String: Any] + + init?(raw: Any) { + guard let dict = raw as? [String: Any] else { return nil } + self.raw = dict + } + + var title: String? { self.raw["title"] as? String } + var description: String? { self.raw["description"] as? String } + var enumValues: [Any]? { self.raw["enum"] as? [Any] } + var constValue: Any? { self.raw["const"] } + var explicitDefault: Any? { self.raw["default"] } + var requiredKeys: Set { + Set((self.raw["required"] as? [String]) ?? []) + } + + var typeList: [String] { + if let type = self.raw["type"] as? String { return [type] } + if let types = self.raw["type"] as? [String] { return types } + return [] + } + + var schemaType: String? { + let filtered = self.typeList.filter { $0 != "null" } + if let first = filtered.first { return first } + return self.typeList.first + } + + var isNullSchema: Bool { + let types = self.typeList + return types.count == 1 && types.first == "null" + } + + var properties: [String: ConfigSchemaNode] { + guard let props = self.raw["properties"] as? [String: Any] else { return [:] } + return props.compactMapValues { ConfigSchemaNode(raw: $0) } + } + + var anyOf: [ConfigSchemaNode] { + guard let raw = self.raw["anyOf"] as? [Any] else { return [] } + return raw.compactMap { ConfigSchemaNode(raw: $0) } + } + + var oneOf: [ConfigSchemaNode] { + guard let raw = self.raw["oneOf"] as? [Any] else { return [] } + return raw.compactMap { ConfigSchemaNode(raw: $0) } + } + + var literalValue: Any? { + if let constValue { return constValue } + if let enumValues, enumValues.count == 1 { return enumValues[0] } + return nil + } + + var items: ConfigSchemaNode? { + if let items = self.raw["items"] as? [Any], let first = items.first { + return ConfigSchemaNode(raw: first) + } + if let items = self.raw["items"] { + return ConfigSchemaNode(raw: items) + } + return nil + } + + var additionalProperties: ConfigSchemaNode? { + if let additional = self.raw["additionalProperties"] as? [String: Any] { + return ConfigSchemaNode(raw: additional) + } + return nil + } + + var allowsAdditionalProperties: Bool { + if let allow = self.raw["additionalProperties"] as? Bool { return allow } + return self.additionalProperties != nil + } + + var defaultValue: Any { + if let value = self.raw["default"] { return value } + switch self.schemaType { + case "object": + return [String: Any]() + case "array": + return [Any]() + case "boolean": + return false + case "integer": + return 0 + case "number": + return 0.0 + case "string": + return "" + default: + return "" + } + } + + func node(at path: ConfigPath) -> ConfigSchemaNode? { + var current: ConfigSchemaNode? = self + for segment in path { + guard let node = current else { return nil } + switch segment { + case let .key(key): + if node.schemaType == "object" { + if let next = node.properties[key] { + current = next + continue + } + if let additional = node.additionalProperties { + current = additional + continue + } + return nil + } + return nil + case .index: + guard node.schemaType == "array" else { return nil } + current = node.items + } + } + return current + } +} + +func decodeUiHints(_ raw: [String: Any]) -> [String: ConfigUiHint] { + raw.reduce(into: [:]) { result, entry in + if let hint = entry.value as? [String: Any] { + result[entry.key] = ConfigUiHint(raw: hint) + } + } +} + +func hintForPath(_ path: ConfigPath, hints: [String: ConfigUiHint]) -> ConfigUiHint? { + let key = pathKey(path) + if let direct = hints[key] { return direct } + let segments = key.split(separator: ".").map(String.init) + for (hintKey, hint) in hints { + guard hintKey.contains("*") else { continue } + let hintSegments = hintKey.split(separator: ".").map(String.init) + guard hintSegments.count == segments.count else { continue } + var match = true + for (index, seg) in segments.enumerated() { + let hintSegment = hintSegments[index] + if hintSegment != "*", hintSegment != seg { + match = false + break + } + } + if match { return hint } + } + return nil +} + +func isSensitivePath(_ path: ConfigPath) -> Bool { + let key = pathKey(path).lowercased() + return key.contains("token") + || key.contains("password") + || key.contains("secret") + || key.contains("apikey") + || key.hasSuffix("key") +} + +func pathKey(_ path: ConfigPath) -> String { + path.compactMap { segment -> String? in + switch segment { + case let .key(key): return key + case .index: return nil + } + } + .joined(separator: ".") +} diff --git a/apps/macos/Sources/OpenClaw/ConfigSettings.swift b/apps/macos/Sources/OpenClaw/ConfigSettings.swift new file mode 100644 index 0000000000000000000000000000000000000000..f64a6bce94ebdfe5a7a8e91ab57354e119aab5bf --- /dev/null +++ b/apps/macos/Sources/OpenClaw/ConfigSettings.swift @@ -0,0 +1,391 @@ +import SwiftUI + +@MainActor +struct ConfigSettings: View { + private let isPreview = ProcessInfo.processInfo.isPreview + private let isNixMode = ProcessInfo.processInfo.isNixMode + @Bindable var store: ChannelsStore + @State private var hasLoaded = false + @State private var activeSectionKey: String? + @State private var activeSubsection: SubsectionSelection? + + init(store: ChannelsStore = .shared) { + self.store = store + } + + var body: some View { + HStack(spacing: 16) { + self.sidebar + self.detail + } + .frame(maxWidth: .infinity, maxHeight: .infinity, alignment: .topLeading) + .task { + guard !self.hasLoaded else { return } + guard !self.isPreview else { return } + self.hasLoaded = true + await self.store.loadConfigSchema() + await self.store.loadConfig() + } + .onAppear { self.ensureSelection() } + .onChange(of: self.store.configSchemaLoading) { _, loading in + if !loading { self.ensureSelection() } + } + } +} + +extension ConfigSettings { + private enum SubsectionSelection: Hashable { + case all + case key(String) + } + + private struct ConfigSection: Identifiable { + let key: String + let label: String + let help: String? + let node: ConfigSchemaNode + + var id: String { self.key } + } + + private struct ConfigSubsection: Identifiable { + let key: String + let label: String + let help: String? + let node: ConfigSchemaNode + let path: ConfigPath + + var id: String { self.key } + } + + private var sections: [ConfigSection] { + guard let schema = self.store.configSchema else { return [] } + return self.resolveSections(schema) + } + + private var activeSection: ConfigSection? { + self.sections.first { $0.key == self.activeSectionKey } + } + + private var sidebar: some View { + ScrollView { + LazyVStack(alignment: .leading, spacing: 8) { + if self.sections.isEmpty { + Text("No config sections available.") + .font(.caption) + .foregroundStyle(.secondary) + .padding(.horizontal, 6) + .padding(.vertical, 4) + } else { + ForEach(self.sections) { section in + self.sidebarRow(section) + } + } + } + .padding(.vertical, 10) + .padding(.horizontal, 10) + } + .frame(minWidth: 220, idealWidth: 240, maxWidth: 280, maxHeight: .infinity, alignment: .topLeading) + .background( + RoundedRectangle(cornerRadius: 12, style: .continuous) + .fill(Color(nsColor: .windowBackgroundColor))) + .clipShape(RoundedRectangle(cornerRadius: 12, style: .continuous)) + } + + private var detail: some View { + VStack(alignment: .leading, spacing: 16) { + if self.store.configSchemaLoading { + ProgressView().controlSize(.small) + } else if let section = self.activeSection { + self.sectionDetail(section) + } else if self.store.configSchema != nil { + self.emptyDetail + } else { + Text("Schema unavailable.") + .font(.caption) + .foregroundStyle(.secondary) + } + } + .frame(minWidth: 460, maxWidth: .infinity, maxHeight: .infinity, alignment: .topLeading) + } + + private var emptyDetail: some View { + VStack(alignment: .leading, spacing: 8) { + self.header + Text("Select a config section to view settings.") + .font(.callout) + .foregroundStyle(.secondary) + } + .padding(.horizontal, 24) + .padding(.vertical, 18) + } + + private func sectionDetail(_ section: ConfigSection) -> some View { + ScrollView(.vertical) { + VStack(alignment: .leading, spacing: 16) { + self.header + if let status = self.store.configStatus { + Text(status) + .font(.callout) + .foregroundStyle(.secondary) + } + self.actionRow + self.sectionHeader(section) + self.subsectionNav(section) + self.sectionForm(section) + if self.store.configDirty, !self.isNixMode { + Text("Unsaved changes") + .font(.caption) + .foregroundStyle(.secondary) + } + Spacer(minLength: 0) + } + .frame(maxWidth: .infinity, alignment: .leading) + .padding(.horizontal, 24) + .padding(.vertical, 18) + .groupBoxStyle(PlainSettingsGroupBoxStyle()) + } + } + + @ViewBuilder + private var header: some View { + Text("Config") + .font(.title3.weight(.semibold)) + Text(self.isNixMode + ? "This tab is read-only in Nix mode. Edit config via Nix and rebuild." + : "Edit ~/.openclaw/openclaw.json using the schema-driven form.") + .font(.callout) + .foregroundStyle(.secondary) + } + + private func sectionHeader(_ section: ConfigSection) -> some View { + VStack(alignment: .leading, spacing: 6) { + Text(section.label) + .font(.title3.weight(.semibold)) + if let help = section.help { + Text(help) + .font(.callout) + .foregroundStyle(.secondary) + } + } + } + + private var actionRow: some View { + HStack(spacing: 10) { + Button("Reload") { + Task { await self.store.reloadConfigDraft() } + } + .disabled(!self.store.configLoaded) + + Button(self.store.isSavingConfig ? "Saving…" : "Save") { + Task { await self.store.saveConfigDraft() } + } + .disabled(self.isNixMode || self.store.isSavingConfig || !self.store.configDirty) + } + .buttonStyle(.bordered) + } + + private func sidebarRow(_ section: ConfigSection) -> some View { + let isSelected = self.activeSectionKey == section.key + return Button { + self.selectSection(section) + } label: { + VStack(alignment: .leading, spacing: 2) { + Text(section.label) + if let help = section.help { + Text(help) + .font(.caption) + .foregroundStyle(.secondary) + .lineLimit(2) + } + } + .padding(.vertical, 6) + .padding(.horizontal, 8) + .frame(maxWidth: .infinity, alignment: .leading) + .background(isSelected ? Color.accentColor.opacity(0.18) : Color.clear) + .clipShape(RoundedRectangle(cornerRadius: 10, style: .continuous)) + .background(Color.clear) + .contentShape(Rectangle()) + } + .frame(maxWidth: .infinity, alignment: .leading) + .buttonStyle(.plain) + .contentShape(Rectangle()) + } + + @ViewBuilder + private func subsectionNav(_ section: ConfigSection) -> some View { + let subsections = self.resolveSubsections(for: section) + if subsections.isEmpty { + EmptyView() + } else { + ScrollView(.horizontal, showsIndicators: false) { + HStack(spacing: 8) { + self.subsectionButton( + title: "All", + isSelected: self.activeSubsection == .all) + { + self.activeSubsection = .all + } + ForEach(subsections) { subsection in + self.subsectionButton( + title: subsection.label, + isSelected: self.activeSubsection == .key(subsection.key)) + { + self.activeSubsection = .key(subsection.key) + } + } + } + .padding(.vertical, 2) + } + } + } + + private func subsectionButton( + title: String, + isSelected: Bool, + action: @escaping () -> Void) -> some View + { + Button(action: action) { + Text(title) + .font(.callout.weight(.semibold)) + .foregroundStyle(isSelected ? Color.accentColor : .primary) + .padding(.horizontal, 10) + .padding(.vertical, 6) + .background(isSelected ? Color.accentColor.opacity(0.18) : Color(nsColor: .controlBackgroundColor)) + .clipShape(Capsule()) + } + .buttonStyle(.plain) + } + + private func sectionForm(_ section: ConfigSection) -> some View { + let subsection = self.activeSubsection + let defaultPath: ConfigPath = [.key(section.key)] + let subsections = self.resolveSubsections(for: section) + let resolved: (ConfigSchemaNode, ConfigPath) = { + if case let .key(key) = subsection, + let match = subsections.first(where: { $0.key == key }) + { + return (match.node, match.path) + } + return (self.resolvedSchemaNode(section.node), defaultPath) + }() + + return ConfigSchemaForm(store: self.store, schema: resolved.0, path: resolved.1) + .disabled(self.isNixMode) + } + + private func ensureSelection() { + guard let schema = self.store.configSchema else { return } + let sections = self.resolveSections(schema) + guard !sections.isEmpty else { return } + + let active = sections.first { $0.key == self.activeSectionKey } ?? sections[0] + if self.activeSectionKey != active.key { + self.activeSectionKey = active.key + } + self.ensureSubsection(for: active) + } + + private func ensureSubsection(for section: ConfigSection) { + let subsections = self.resolveSubsections(for: section) + guard !subsections.isEmpty else { + self.activeSubsection = nil + return + } + + switch self.activeSubsection { + case .all: + return + case let .key(key): + if subsections.contains(where: { $0.key == key }) { return } + case .none: + break + } + + if let first = subsections.first { + self.activeSubsection = .key(first.key) + } + } + + private func selectSection(_ section: ConfigSection) { + guard self.activeSectionKey != section.key else { return } + self.activeSectionKey = section.key + let subsections = self.resolveSubsections(for: section) + if let first = subsections.first { + self.activeSubsection = .key(first.key) + } else { + self.activeSubsection = nil + } + } + + private func resolveSections(_ root: ConfigSchemaNode) -> [ConfigSection] { + let node = self.resolvedSchemaNode(root) + let hints = self.store.configUiHints + let keys = node.properties.keys.sorted { lhs, rhs in + let orderA = hintForPath([.key(lhs)], hints: hints)?.order ?? 0 + let orderB = hintForPath([.key(rhs)], hints: hints)?.order ?? 0 + if orderA != orderB { return orderA < orderB } + return lhs < rhs + } + + return keys.compactMap { key in + guard let child = node.properties[key] else { return nil } + let path: ConfigPath = [.key(key)] + let hint = hintForPath(path, hints: hints) + let label = hint?.label + ?? child.title + ?? self.humanize(key) + let help = hint?.help ?? child.description + return ConfigSection(key: key, label: label, help: help, node: child) + } + } + + private func resolveSubsections(for section: ConfigSection) -> [ConfigSubsection] { + let node = self.resolvedSchemaNode(section.node) + guard node.schemaType == "object" else { return [] } + let hints = self.store.configUiHints + let keys = node.properties.keys.sorted { lhs, rhs in + let orderA = hintForPath([.key(section.key), .key(lhs)], hints: hints)?.order ?? 0 + let orderB = hintForPath([.key(section.key), .key(rhs)], hints: hints)?.order ?? 0 + if orderA != orderB { return orderA < orderB } + return lhs < rhs + } + + return keys.compactMap { key in + guard let child = node.properties[key] else { return nil } + let path: ConfigPath = [.key(section.key), .key(key)] + let hint = hintForPath(path, hints: hints) + let label = hint?.label + ?? child.title + ?? self.humanize(key) + let help = hint?.help ?? child.description + return ConfigSubsection( + key: key, + label: label, + help: help, + node: child, + path: path) + } + } + + private func resolvedSchemaNode(_ node: ConfigSchemaNode) -> ConfigSchemaNode { + let variants = node.anyOf.isEmpty ? node.oneOf : node.anyOf + if !variants.isEmpty { + let nonNull = variants.filter { !$0.isNullSchema } + if nonNull.count == 1, let only = nonNull.first { return only } + } + return node + } + + private func humanize(_ key: String) -> String { + key.replacingOccurrences(of: "_", with: " ") + .replacingOccurrences(of: "-", with: " ") + .capitalized + } +} + +struct ConfigSettings_Previews: PreviewProvider { + static var previews: some View { + ConfigSettings() + } +} diff --git a/apps/macos/Sources/OpenClaw/ConfigStore.swift b/apps/macos/Sources/OpenClaw/ConfigStore.swift new file mode 100644 index 0000000000000000000000000000000000000000..4e9437ff86eb43e11ccd3115bb2c5eae7300e4d8 --- /dev/null +++ b/apps/macos/Sources/OpenClaw/ConfigStore.swift @@ -0,0 +1,117 @@ +import OpenClawProtocol +import Foundation + +enum ConfigStore { + struct Overrides: Sendable { + var isRemoteMode: (@Sendable () async -> Bool)? + var loadLocal: (@MainActor @Sendable () -> [String: Any])? + var saveLocal: (@MainActor @Sendable ([String: Any]) -> Void)? + var loadRemote: (@MainActor @Sendable () async -> [String: Any])? + var saveRemote: (@MainActor @Sendable ([String: Any]) async throws -> Void)? + } + + private actor OverrideStore { + var overrides = Overrides() + + func setOverride(_ overrides: Overrides) { + self.overrides = overrides + } + } + + private static let overrideStore = OverrideStore() + @MainActor private static var lastHash: String? + + private static func isRemoteMode() async -> Bool { + let overrides = await self.overrideStore.overrides + if let override = overrides.isRemoteMode { + return await override() + } + return await MainActor.run { AppStateStore.shared.connectionMode == .remote } + } + + @MainActor + static func load() async -> [String: Any] { + let overrides = await self.overrideStore.overrides + if await self.isRemoteMode() { + if let override = overrides.loadRemote { + return await override() + } + return await self.loadFromGateway() ?? [:] + } + if let override = overrides.loadLocal { + return override() + } + if let gateway = await self.loadFromGateway() { + return gateway + } + return OpenClawConfigFile.loadDict() + } + + @MainActor + static func save(_ root: sending [String: Any]) async throws { + let overrides = await self.overrideStore.overrides + if await self.isRemoteMode() { + if let override = overrides.saveRemote { + try await override(root) + } else { + try await self.saveToGateway(root) + } + } else { + if let override = overrides.saveLocal { + override(root) + } else { + do { + try await self.saveToGateway(root) + } catch { + OpenClawConfigFile.saveDict(root) + } + } + } + } + + @MainActor + private static func loadFromGateway() async -> [String: Any]? { + do { + let snap: ConfigSnapshot = try await GatewayConnection.shared.requestDecoded( + method: .configGet, + params: nil, + timeoutMs: 8000) + self.lastHash = snap.hash + return snap.config?.mapValues { $0.foundationValue } ?? [:] + } catch { + return nil + } + } + + @MainActor + private static func saveToGateway(_ root: [String: Any]) async throws { + if self.lastHash == nil { + _ = await self.loadFromGateway() + } + let data = try JSONSerialization.data(withJSONObject: root, options: [.prettyPrinted, .sortedKeys]) + guard let raw = String(data: data, encoding: .utf8) else { + throw NSError(domain: "ConfigStore", code: 1, userInfo: [ + NSLocalizedDescriptionKey: "Failed to encode config.", + ]) + } + var params: [String: AnyCodable] = ["raw": AnyCodable(raw)] + if let baseHash = self.lastHash { + params["baseHash"] = AnyCodable(baseHash) + } + _ = try await GatewayConnection.shared.requestRaw( + method: .configSet, + params: params, + timeoutMs: 10000) + _ = await self.loadFromGateway() + } + + #if DEBUG + static func _testSetOverrides(_ overrides: Overrides) async { + await self.overrideStore.setOverride(overrides) + } + + static func _testClearOverrides() async { + await self.overrideStore.setOverride(.init()) + } + #endif +} diff --git a/apps/macos/Sources/OpenClaw/ConnectionModeCoordinator.swift b/apps/macos/Sources/OpenClaw/ConnectionModeCoordinator.swift new file mode 100644 index 0000000000000000000000000000000000000000..b1c5eab1dbb405c51c2b53089871834ba5ae511e --- /dev/null +++ b/apps/macos/Sources/OpenClaw/ConnectionModeCoordinator.swift @@ -0,0 +1,79 @@ +import Foundation +import OSLog + +@MainActor +final class ConnectionModeCoordinator { + static let shared = ConnectionModeCoordinator() + + private let logger = Logger(subsystem: "ai.openclaw", category: "connection") + private var lastMode: AppState.ConnectionMode? + + /// Apply the requested connection mode by starting/stopping local gateway, + /// managing the control-channel SSH tunnel, and cleaning up chat windows/panels. + func apply(mode: AppState.ConnectionMode, paused: Bool) async { + if let lastMode = self.lastMode, lastMode != mode { + GatewayProcessManager.shared.clearLastFailure() + NodesStore.shared.lastError = nil + } + self.lastMode = mode + switch mode { + case .unconfigured: + _ = await NodeServiceManager.stop() + NodesStore.shared.lastError = nil + await RemoteTunnelManager.shared.stopAll() + WebChatManager.shared.resetTunnels() + GatewayProcessManager.shared.stop() + await GatewayConnection.shared.shutdown() + await ControlChannel.shared.disconnect() + Task.detached { await PortGuardian.shared.sweep(mode: .unconfigured) } + + case .local: + _ = await NodeServiceManager.stop() + NodesStore.shared.lastError = nil + await RemoteTunnelManager.shared.stopAll() + WebChatManager.shared.resetTunnels() + let shouldStart = GatewayAutostartPolicy.shouldStartGateway(mode: .local, paused: paused) + if shouldStart { + GatewayProcessManager.shared.setActive(true) + if GatewayAutostartPolicy.shouldEnsureLaunchAgent( + mode: .local, + paused: paused) + { + Task { await GatewayProcessManager.shared.ensureLaunchAgentEnabledIfNeeded() } + } + _ = await GatewayProcessManager.shared.waitForGatewayReady() + } else { + GatewayProcessManager.shared.stop() + } + do { + try await ControlChannel.shared.configure(mode: .local) + } catch { + // Control channel will mark itself degraded; nothing else to do here. + self.logger.error( + "control channel local configure failed: \(error.localizedDescription, privacy: .public)") + } + Task.detached { await PortGuardian.shared.sweep(mode: .local) } + + case .remote: + // Never run a local gateway in remote mode. + GatewayProcessManager.shared.stop() + WebChatManager.shared.resetTunnels() + + do { + NodesStore.shared.lastError = nil + if let error = await NodeServiceManager.start() { + NodesStore.shared.lastError = "Node service start failed: \(error)" + } + _ = try await GatewayEndpointStore.shared.ensureRemoteControlTunnel() + let settings = CommandResolver.connectionSettings() + try await ControlChannel.shared.configure(mode: .remote( + target: settings.target, + identity: settings.identity)) + } catch { + self.logger.error("remote tunnel/configure failed: \(error.localizedDescription, privacy: .public)") + } + + Task.detached { await PortGuardian.shared.sweep(mode: .remote) } + } + } +} diff --git a/apps/macos/Sources/OpenClaw/ConnectionModeResolver.swift b/apps/macos/Sources/OpenClaw/ConnectionModeResolver.swift new file mode 100644 index 0000000000000000000000000000000000000000..60c6fab9d560829c2d18f6fc644e4f7f1f4c7992 --- /dev/null +++ b/apps/macos/Sources/OpenClaw/ConnectionModeResolver.swift @@ -0,0 +1,49 @@ +import Foundation + +enum EffectiveConnectionModeSource: Sendable, Equatable { + case configMode + case configRemoteURL + case userDefaults + case onboarding +} + +struct EffectiveConnectionMode: Sendable, Equatable { + let mode: AppState.ConnectionMode + let source: EffectiveConnectionModeSource +} + +enum ConnectionModeResolver { + static func resolve( + root: [String: Any], + defaults: UserDefaults = .standard) -> EffectiveConnectionMode + { + let gateway = root["gateway"] as? [String: Any] + let configModeRaw = (gateway?["mode"] as? String) ?? "" + let configMode = configModeRaw + .trimmingCharacters(in: .whitespacesAndNewlines) + .lowercased() + + switch configMode { + case "local": + return EffectiveConnectionMode(mode: .local, source: .configMode) + case "remote": + return EffectiveConnectionMode(mode: .remote, source: .configMode) + default: + break + } + + let remoteURLRaw = ((gateway?["remote"] as? [String: Any])?["url"] as? String) ?? "" + let remoteURL = remoteURLRaw.trimmingCharacters(in: .whitespacesAndNewlines) + if !remoteURL.isEmpty { + return EffectiveConnectionMode(mode: .remote, source: .configRemoteURL) + } + + if let storedModeRaw = defaults.string(forKey: connectionModeKey) { + let storedMode = AppState.ConnectionMode(rawValue: storedModeRaw) ?? .local + return EffectiveConnectionMode(mode: storedMode, source: .userDefaults) + } + + let seen = defaults.bool(forKey: "openclaw.onboardingSeen") + return EffectiveConnectionMode(mode: seen ? .local : .unconfigured, source: .onboarding) + } +} diff --git a/apps/macos/Sources/OpenClaw/Constants.swift b/apps/macos/Sources/OpenClaw/Constants.swift new file mode 100644 index 0000000000000000000000000000000000000000..40a0acbbacc086dbb29694a774fcf974d3ada35d --- /dev/null +++ b/apps/macos/Sources/OpenClaw/Constants.swift @@ -0,0 +1,46 @@ +import Foundation + +let launchdLabel = "ai.openclaw.mac" +let gatewayLaunchdLabel = "ai.openclaw.gateway" +let onboardingVersionKey = "openclaw.onboardingVersion" +let onboardingSeenKey = "openclaw.onboardingSeen" +let currentOnboardingVersion = 7 +let pauseDefaultsKey = "openclaw.pauseEnabled" +let iconAnimationsEnabledKey = "openclaw.iconAnimationsEnabled" +let swabbleEnabledKey = "openclaw.swabbleEnabled" +let swabbleTriggersKey = "openclaw.swabbleTriggers" +let voiceWakeTriggerChimeKey = "openclaw.voiceWakeTriggerChime" +let voiceWakeSendChimeKey = "openclaw.voiceWakeSendChime" +let showDockIconKey = "openclaw.showDockIcon" +let defaultVoiceWakeTriggers = ["openclaw"] +let voiceWakeMaxWords = 32 +let voiceWakeMaxWordLength = 64 +let voiceWakeMicKey = "openclaw.voiceWakeMicID" +let voiceWakeMicNameKey = "openclaw.voiceWakeMicName" +let voiceWakeLocaleKey = "openclaw.voiceWakeLocaleID" +let voiceWakeAdditionalLocalesKey = "openclaw.voiceWakeAdditionalLocaleIDs" +let voicePushToTalkEnabledKey = "openclaw.voicePushToTalkEnabled" +let talkEnabledKey = "openclaw.talkEnabled" +let iconOverrideKey = "openclaw.iconOverride" +let connectionModeKey = "openclaw.connectionMode" +let remoteTargetKey = "openclaw.remoteTarget" +let remoteIdentityKey = "openclaw.remoteIdentity" +let remoteProjectRootKey = "openclaw.remoteProjectRoot" +let remoteCliPathKey = "openclaw.remoteCliPath" +let canvasEnabledKey = "openclaw.canvasEnabled" +let cameraEnabledKey = "openclaw.cameraEnabled" +let systemRunPolicyKey = "openclaw.systemRunPolicy" +let systemRunAllowlistKey = "openclaw.systemRunAllowlist" +let systemRunEnabledKey = "openclaw.systemRunEnabled" +let locationModeKey = "openclaw.locationMode" +let locationPreciseKey = "openclaw.locationPreciseEnabled" +let peekabooBridgeEnabledKey = "openclaw.peekabooBridgeEnabled" +let deepLinkKeyKey = "openclaw.deepLinkKey" +let modelCatalogPathKey = "openclaw.modelCatalogPath" +let modelCatalogReloadKey = "openclaw.modelCatalogReload" +let cliInstallPromptedVersionKey = "openclaw.cliInstallPromptedVersion" +let heartbeatsEnabledKey = "openclaw.heartbeatsEnabled" +let debugPaneEnabledKey = "openclaw.debugPaneEnabled" +let debugFileLogEnabledKey = "openclaw.debug.fileLogEnabled" +let appLogLevelKey = "openclaw.debug.appLogLevel" +let voiceWakeSupported: Bool = ProcessInfo.processInfo.operatingSystemVersion.majorVersion >= 26 diff --git a/apps/macos/Sources/OpenClaw/ContextMenuCardView.swift b/apps/macos/Sources/OpenClaw/ContextMenuCardView.swift new file mode 100644 index 0000000000000000000000000000000000000000..41005e8260e416280be3feea6c945e06cf0cca8f --- /dev/null +++ b/apps/macos/Sources/OpenClaw/ContextMenuCardView.swift @@ -0,0 +1,121 @@ +import Foundation +import SwiftUI + +/// Context usage card shown at the top of the menubar menu. +struct ContextMenuCardView: View { + private let rows: [SessionRow] + private let statusText: String? + private let isLoading: Bool + private let paddingTop: CGFloat = 8 + private let paddingBottom: CGFloat = 8 + private let paddingTrailing: CGFloat = 10 + private let paddingLeading: CGFloat = 20 + private let barHeight: CGFloat = 3 + + init( + rows: [SessionRow], + statusText: String? = nil, + isLoading: Bool = false) + { + self.rows = rows + self.statusText = statusText + self.isLoading = isLoading + } + + var body: some View { + VStack(alignment: .leading, spacing: 6) { + HStack(alignment: .firstTextBaseline) { + Text("Context") + .font(.caption.weight(.semibold)) + .foregroundStyle(.secondary) + Spacer(minLength: 10) + Text(self.subtitle) + .font(.caption) + .foregroundStyle(.secondary) + } + + if let statusText { + Text(statusText) + .font(.caption) + .foregroundStyle(.secondary) + } else if self.rows.isEmpty, !self.isLoading { + Text("No active sessions") + .font(.caption) + .foregroundStyle(.secondary) + } else { + VStack(alignment: .leading, spacing: 12) { + if self.rows.isEmpty, self.isLoading { + ForEach(0..<2, id: \.self) { _ in + self.placeholderRow + } + } else { + ForEach(self.rows) { row in + self.sessionRow(row) + } + } + } + } + } + .padding(.top, self.paddingTop) + .padding(.bottom, self.paddingBottom) + .padding(.leading, self.paddingLeading) + .padding(.trailing, self.paddingTrailing) + .frame(minWidth: 300, maxWidth: .infinity, alignment: .leading) + .transaction { txn in txn.animation = nil } + } + + private var subtitle: String { + let count = self.rows.count + if count == 1 { return "1 session · 24h" } + return "\(count) sessions · 24h" + } + + @ViewBuilder + private func sessionRow(_ row: SessionRow) -> some View { + VStack(alignment: .leading, spacing: 5) { + ContextUsageBar( + usedTokens: row.tokens.total, + contextTokens: row.tokens.contextTokens, + height: self.barHeight) + + HStack(alignment: .firstTextBaseline, spacing: 8) { + Text(row.label) + .font(.caption.weight(row.key == "main" ? .semibold : .regular)) + .lineLimit(1) + .truncationMode(.middle) + .layoutPriority(1) + Spacer(minLength: 8) + Text(row.tokens.contextSummaryShort) + .font(.caption.monospacedDigit()) + .foregroundStyle(.secondary) + .lineLimit(1) + .fixedSize(horizontal: true, vertical: false) + .layoutPriority(2) + } + } + .padding(.vertical, 2) + } + + private var placeholderRow: some View { + VStack(alignment: .leading, spacing: 5) { + ContextUsageBar( + usedTokens: 0, + contextTokens: 200_000, + height: self.barHeight) + + HStack(alignment: .firstTextBaseline, spacing: 8) { + Text("main") + .font(.caption.weight(.semibold)) + .lineLimit(1) + .layoutPriority(1) + Spacer(minLength: 8) + Text("000k/000k") + .font(.caption.monospacedDigit()) + .foregroundStyle(.secondary) + .fixedSize(horizontal: true, vertical: false) + .layoutPriority(2) + } + .redacted(reason: .placeholder) + } + } +} diff --git a/apps/macos/Sources/OpenClaw/ContextUsageBar.swift b/apps/macos/Sources/OpenClaw/ContextUsageBar.swift new file mode 100644 index 0000000000000000000000000000000000000000..f5bfa0530b0611771b837352a23eaf9e968d5d6b --- /dev/null +++ b/apps/macos/Sources/OpenClaw/ContextUsageBar.swift @@ -0,0 +1,93 @@ +import SwiftUI + +struct ContextUsageBar: View { + let usedTokens: Int + let contextTokens: Int + var width: CGFloat? + var height: CGFloat = 6 + + private static let okGreen: NSColor = .init(name: nil) { appearance in + let base = NSColor.systemGreen + let match = appearance.bestMatch(from: [.aqua, .darkAqua]) + if match == .darkAqua { return base } + return base.blended(withFraction: 0.24, of: .black) ?? base + } + + private static let trackFill: NSColor = .init(name: nil) { appearance in + let match = appearance.bestMatch(from: [.aqua, .darkAqua]) + if match == .darkAqua { return NSColor.white.withAlphaComponent(0.14) } + return NSColor.black.withAlphaComponent(0.12) + } + + private static let trackStroke: NSColor = .init(name: nil) { appearance in + let match = appearance.bestMatch(from: [.aqua, .darkAqua]) + if match == .darkAqua { return NSColor.white.withAlphaComponent(0.22) } + return NSColor.black.withAlphaComponent(0.2) + } + + private var clampedFractionUsed: Double { + guard self.contextTokens > 0 else { return 0 } + return min(1, max(0, Double(self.usedTokens) / Double(self.contextTokens))) + } + + private var percentUsed: Int? { + guard self.contextTokens > 0, self.usedTokens > 0 else { return nil } + return min(100, Int(round(self.clampedFractionUsed * 100))) + } + + private var tint: Color { + guard let pct = self.percentUsed else { return .secondary } + if pct >= 95 { return Color(nsColor: .systemRed) } + if pct >= 80 { return Color(nsColor: .systemOrange) } + if pct >= 60 { return Color(nsColor: .systemYellow) } + return Color(nsColor: Self.okGreen) + } + + var body: some View { + let fraction = self.clampedFractionUsed + Group { + if let width = self.width, width > 0 { + self.barBody(width: width, fraction: fraction) + .frame(width: width, height: self.height) + } else { + GeometryReader { proxy in + self.barBody(width: proxy.size.width, fraction: fraction) + .frame(width: proxy.size.width, height: self.height) + } + .frame(height: self.height) + } + } + .accessibilityLabel("Context usage") + .accessibilityValue(self.accessibilityValue) + } + + private var accessibilityValue: String { + if self.contextTokens <= 0 { return "Unknown context window" } + let pct = Int(round(self.clampedFractionUsed * 100)) + return "\(pct) percent used" + } + + @ViewBuilder + private func barBody(width: CGFloat, fraction: Double) -> some View { + let radius = self.height / 2 + let trackFill = Color(nsColor: Self.trackFill) + let trackStroke = Color(nsColor: Self.trackStroke) + let fillWidth = max(1, floor(width * CGFloat(fraction))) + + ZStack(alignment: .leading) { + RoundedRectangle(cornerRadius: radius, style: .continuous) + .fill(trackFill) + .overlay { + RoundedRectangle(cornerRadius: radius, style: .continuous) + .strokeBorder(trackStroke, lineWidth: 0.75) + } + + RoundedRectangle(cornerRadius: radius, style: .continuous) + .fill(self.tint) + .frame(width: fillWidth) + .mask { + RoundedRectangle(cornerRadius: radius, style: .continuous) + } + } + } +} diff --git a/apps/macos/Sources/OpenClaw/ControlChannel.swift b/apps/macos/Sources/OpenClaw/ControlChannel.swift new file mode 100644 index 0000000000000000000000000000000000000000..9436b22ecb84864e8b20796520e6a3b4729b9c3b --- /dev/null +++ b/apps/macos/Sources/OpenClaw/ControlChannel.swift @@ -0,0 +1,425 @@ +import OpenClawKit +import OpenClawProtocol +import Foundation +import Observation +import SwiftUI + +struct ControlHeartbeatEvent: Codable { + let ts: Double + let status: String + let to: String? + let preview: String? + let durationMs: Double? + let hasMedia: Bool? + let reason: String? +} + +struct ControlAgentEvent: Codable, Sendable, Identifiable { + var id: String { "\(self.runId)-\(self.seq)" } + let runId: String + let seq: Int + let stream: String + let ts: Double + let data: [String: OpenClawProtocol.AnyCodable] + let summary: String? +} + +enum ControlChannelError: Error, LocalizedError { + case disconnected + case badResponse(String) + + var errorDescription: String? { + switch self { + case .disconnected: "Control channel disconnected" + case let .badResponse(msg): msg + } + } +} + +@MainActor +@Observable +final class ControlChannel { + static let shared = ControlChannel() + + enum Mode { + case local + case remote(target: String, identity: String) + } + + enum ConnectionState: Equatable { + case disconnected + case connecting + case connected + case degraded(String) + } + + private(set) var state: ConnectionState = .disconnected { + didSet { + CanvasManager.shared.refreshDebugStatus() + guard oldValue != self.state else { return } + switch self.state { + case .connected: + self.logger.info("control channel state -> connected") + case .connecting: + self.logger.info("control channel state -> connecting") + case .disconnected: + self.logger.info("control channel state -> disconnected") + self.scheduleRecovery(reason: "disconnected") + case let .degraded(message): + let detail = message.isEmpty ? "degraded" : "degraded: \(message)" + self.logger.info("control channel state -> \(detail, privacy: .public)") + self.scheduleRecovery(reason: message) + } + } + } + + private(set) var lastPingMs: Double? + private(set) var authSourceLabel: String? + + private let logger = Logger(subsystem: "ai.openclaw", category: "control") + + private var eventTask: Task? + private var recoveryTask: Task? + private var lastRecoveryAt: Date? + + private init() { + self.startEventStream() + } + + func configure() async { + self.logger.info("control channel configure mode=local") + await self.refreshEndpoint(reason: "configure") + } + + func configure(mode: Mode = .local) async throws { + switch mode { + case .local: + await self.configure() + case let .remote(target, identity): + do { + _ = (target, identity) + let idSet = !identity.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty + self.logger.info( + "control channel configure mode=remote " + + "target=\(target, privacy: .public) identitySet=\(idSet, privacy: .public)") + self.state = .connecting + _ = try await GatewayEndpointStore.shared.ensureRemoteControlTunnel() + await self.refreshEndpoint(reason: "configure") + } catch { + self.state = .degraded(error.localizedDescription) + throw error + } + } + } + + func refreshEndpoint(reason: String) async { + self.logger.info("control channel refresh endpoint reason=\(reason, privacy: .public)") + self.state = .connecting + do { + try await self.establishGatewayConnection() + self.state = .connected + PresenceReporter.shared.sendImmediate(reason: "connect") + } catch { + let message = self.friendlyGatewayMessage(error) + self.state = .degraded(message) + } + } + + func disconnect() async { + await GatewayConnection.shared.shutdown() + self.state = .disconnected + self.lastPingMs = nil + self.authSourceLabel = nil + } + + func health(timeout: TimeInterval? = nil) async throws -> Data { + do { + let start = Date() + var params: [String: AnyHashable]? + if let timeout { + params = ["timeout": AnyHashable(Int(timeout * 1000))] + } + let timeoutMs = (timeout ?? 15) * 1000 + let payload = try await self.request(method: "health", params: params, timeoutMs: timeoutMs) + let ms = Date().timeIntervalSince(start) * 1000 + self.lastPingMs = ms + self.state = .connected + return payload + } catch { + let message = self.friendlyGatewayMessage(error) + self.state = .degraded(message) + throw ControlChannelError.badResponse(message) + } + } + + func lastHeartbeat() async throws -> ControlHeartbeatEvent? { + let data = try await self.request(method: "last-heartbeat") + return try JSONDecoder().decode(ControlHeartbeatEvent?.self, from: data) + } + + func request( + method: String, + params: [String: AnyHashable]? = nil, + timeoutMs: Double? = nil) async throws -> Data + { + do { + let rawParams = params?.reduce(into: [String: OpenClawKit.AnyCodable]()) { + $0[$1.key] = OpenClawKit.AnyCodable($1.value.base) + } + let data = try await GatewayConnection.shared.request( + method: method, + params: rawParams, + timeoutMs: timeoutMs) + self.state = .connected + return data + } catch { + let message = self.friendlyGatewayMessage(error) + self.state = .degraded(message) + throw ControlChannelError.badResponse(message) + } + } + + private func friendlyGatewayMessage(_ error: Error) -> String { + // Map URLSession/WS errors into user-facing, actionable text. + if let ctrlErr = error as? ControlChannelError, let desc = ctrlErr.errorDescription { + return desc + } + + // If the gateway explicitly rejects the hello (e.g., auth/token mismatch), surface it. + if let urlErr = error as? URLError, + urlErr.code == .dataNotAllowed // used for WS close 1008 auth failures + { + let reason = urlErr.failureURLString ?? urlErr.localizedDescription + let tokenKey = CommandResolver.connectionModeIsRemote() + ? "gateway.remote.token" + : "gateway.auth.token" + return + "Gateway rejected token; set \(tokenKey) or clear it on the gateway. Reason: \(reason)" + } + + // Common misfire: we connected to the configured localhost port but it is occupied + // by some other process (e.g. a local dev gateway or a stuck SSH forward). + // The gateway handshake returns something we can't parse, which currently + // surfaces as "hello failed (unexpected response)". Give the user a pointer + // to free the port instead of a vague message. + let nsError = error as NSError + if nsError.domain == "Gateway", + nsError.localizedDescription.contains("hello failed (unexpected response)") + { + let port = GatewayEnvironment.gatewayPort() + return """ + Gateway handshake got non-gateway data on localhost:\(port). + Another process is using that port or the SSH forward failed. + Stop the local gateway/port-forward on \(port) and retry Remote mode. + """ + } + + if let urlError = error as? URLError { + let port = GatewayEnvironment.gatewayPort() + switch urlError.code { + case .cancelled: + return "Gateway connection was closed; start the gateway (localhost:\(port)) and retry." + case .cannotFindHost, .cannotConnectToHost: + let isRemote = CommandResolver.connectionModeIsRemote() + if isRemote { + return """ + Cannot reach gateway at localhost:\(port). + Remote mode uses an SSH tunnel—check the SSH target and that the tunnel is running. + """ + } + return "Cannot reach gateway at localhost:\(port); ensure the gateway is running." + case .networkConnectionLost: + return "Gateway connection dropped; gateway likely restarted—retry." + case .timedOut: + return "Gateway request timed out; check gateway on localhost:\(port)." + case .notConnectedToInternet: + return "No network connectivity; cannot reach gateway." + default: + break + } + } + + if nsError.domain == "Gateway", nsError.code == 5 { + let port = GatewayEnvironment.gatewayPort() + return "Gateway request timed out; check the gateway process on localhost:\(port)." + } + + let detail = nsError.localizedDescription.isEmpty ? "unknown gateway error" : nsError.localizedDescription + let trimmed = detail.trimmingCharacters(in: .whitespacesAndNewlines) + if trimmed.lowercased().hasPrefix("gateway error:") { return trimmed } + return "Gateway error: \(trimmed)" + } + + private func scheduleRecovery(reason: String) { + let now = Date() + if let last = self.lastRecoveryAt, now.timeIntervalSince(last) < 10 { return } + guard self.recoveryTask == nil else { return } + self.lastRecoveryAt = now + + self.recoveryTask = Task { [weak self] in + guard let self else { return } + let mode = await MainActor.run { AppStateStore.shared.connectionMode } + guard mode != .unconfigured else { + self.recoveryTask = nil + return + } + + let trimmedReason = reason.trimmingCharacters(in: .whitespacesAndNewlines) + let reasonText = trimmedReason.isEmpty ? "unknown" : trimmedReason + self.logger.info( + "control channel recovery starting " + + "mode=\(String(describing: mode), privacy: .public) " + + "reason=\(reasonText, privacy: .public)") + if mode == .local { + GatewayProcessManager.shared.setActive(true) + } + if mode == .remote { + do { + let port = try await GatewayEndpointStore.shared.ensureRemoteControlTunnel() + self.logger.info("control channel recovery ensured SSH tunnel port=\(port, privacy: .public)") + } catch { + self.logger.error( + "control channel recovery tunnel failed \(error.localizedDescription, privacy: .public)") + } + } + + await self.refreshEndpoint(reason: "recovery:\(reasonText)") + if case .connected = self.state { + self.logger.info("control channel recovery finished") + } else if case let .degraded(message) = self.state { + self.logger.error("control channel recovery failed \(message, privacy: .public)") + } + + self.recoveryTask = nil + } + } + + private func establishGatewayConnection(timeoutMs: Int = 5000) async throws { + try await GatewayConnection.shared.refresh() + let ok = try await GatewayConnection.shared.healthOK(timeoutMs: timeoutMs) + if ok == false { + throw NSError( + domain: "Gateway", + code: 0, + userInfo: [NSLocalizedDescriptionKey: "gateway health not ok"]) + } + await self.refreshAuthSourceLabel() + } + + private func refreshAuthSourceLabel() async { + let isRemote = CommandResolver.connectionModeIsRemote() + let authSource = await GatewayConnection.shared.authSource() + self.authSourceLabel = Self.formatAuthSource(authSource, isRemote: isRemote) + } + + private static func formatAuthSource(_ source: GatewayAuthSource?, isRemote: Bool) -> String? { + guard let source else { return nil } + switch source { + case .deviceToken: + return "Auth: device token (paired device)" + case .sharedToken: + return "Auth: shared token (\(isRemote ? "gateway.remote.token" : "gateway.auth.token"))" + case .password: + return "Auth: password (\(isRemote ? "gateway.remote.password" : "gateway.auth.password"))" + case .none: + return "Auth: none" + } + } + + func sendSystemEvent(_ text: String, params: [String: AnyHashable] = [:]) async throws { + var merged = params + merged["text"] = AnyHashable(text) + _ = try await self.request(method: "system-event", params: merged) + } + + private func startEventStream() { + self.eventTask?.cancel() + self.eventTask = Task { [weak self] in + guard let self else { return } + let stream = await GatewayConnection.shared.subscribe() + for await push in stream { + if Task.isCancelled { return } + await MainActor.run { [weak self] in + self?.handle(push: push) + } + } + } + } + + private func handle(push: GatewayPush) { + switch push { + case let .event(evt) where evt.event == "agent": + if let payload = evt.payload, + let agent = try? GatewayPayloadDecoding.decode(payload, as: ControlAgentEvent.self) + { + AgentEventStore.shared.append(agent) + self.routeWorkActivity(from: agent) + } + case let .event(evt) where evt.event == "heartbeat": + if let payload = evt.payload, + let heartbeat = try? GatewayPayloadDecoding.decode(payload, as: ControlHeartbeatEvent.self), + let data = try? JSONEncoder().encode(heartbeat) + { + NotificationCenter.default.post(name: .controlHeartbeat, object: data) + } + case let .event(evt) where evt.event == "shutdown": + self.state = .degraded("gateway shutdown") + case .snapshot: + self.state = .connected + default: + break + } + } + + private func routeWorkActivity(from event: ControlAgentEvent) { + // We currently treat VoiceWake as the "main" session for UI purposes. + // In the future, the gateway can include a sessionKey to distinguish runs. + let sessionKey = (event.data["sessionKey"]?.value as? String) ?? "main" + + switch event.stream.lowercased() { + case "job": + if let state = event.data["state"]?.value as? String { + WorkActivityStore.shared.handleJob(sessionKey: sessionKey, state: state) + } + case "tool": + let phase = event.data["phase"]?.value as? String ?? "" + let name = event.data["name"]?.value as? String + let meta = event.data["meta"]?.value as? String + let args = Self.bridgeToProtocolArgs(event.data["args"]) + WorkActivityStore.shared.handleTool( + sessionKey: sessionKey, + phase: phase, + name: name, + meta: meta, + args: args) + default: + break + } + } + + private static func bridgeToProtocolArgs( + _ value: OpenClawProtocol.AnyCodable?) -> [String: OpenClawProtocol.AnyCodable]? + { + guard let value else { return nil } + if let dict = value.value as? [String: OpenClawProtocol.AnyCodable] { + return dict + } + if let dict = value.value as? [String: OpenClawKit.AnyCodable], + let data = try? JSONEncoder().encode(dict), + let decoded = try? JSONDecoder().decode([String: OpenClawProtocol.AnyCodable].self, from: data) + { + return decoded + } + if let data = try? JSONEncoder().encode(value), + let decoded = try? JSONDecoder().decode([String: OpenClawProtocol.AnyCodable].self, from: data) + { + return decoded + } + return nil + } +} + +extension Notification.Name { + static let controlHeartbeat = Notification.Name("openclaw.control.heartbeat") + static let controlAgentEvent = Notification.Name("openclaw.control.agent") +} diff --git a/apps/macos/Sources/OpenClaw/CostUsageMenuView.swift b/apps/macos/Sources/OpenClaw/CostUsageMenuView.swift new file mode 100644 index 0000000000000000000000000000000000000000..c94a4de3518e80b78ebfa95d569fc899b2facfbf --- /dev/null +++ b/apps/macos/Sources/OpenClaw/CostUsageMenuView.swift @@ -0,0 +1,99 @@ +import Charts +import SwiftUI + +struct CostUsageHistoryMenuView: View { + let summary: GatewayCostUsageSummary + let width: CGFloat + + var body: some View { + VStack(alignment: .leading, spacing: 10) { + self.header + self.chart + self.footer + } + .padding(.horizontal, 12) + .padding(.vertical, 10) + .frame(width: max(1, self.width), alignment: .leading) + } + + private var header: some View { + let todayKey = CostUsageMenuDateParser.format(Date()) + let todayEntry = self.summary.daily.first { $0.date == todayKey } + let todayCost = CostUsageFormatting.formatUsd(todayEntry?.totalCost) ?? "n/a" + let totalCost = CostUsageFormatting.formatUsd(self.summary.totals.totalCost) ?? "n/a" + + return HStack(alignment: .firstTextBaseline, spacing: 12) { + VStack(alignment: .leading, spacing: 2) { + Text("Today") + .font(.caption2) + .foregroundStyle(.secondary) + Text(todayCost) + .font(.system(size: 14, weight: .semibold)) + } + VStack(alignment: .leading, spacing: 2) { + Text("Last \(self.summary.days)d") + .font(.caption2) + .foregroundStyle(.secondary) + Text(totalCost) + .font(.system(size: 14, weight: .semibold)) + } + Spacer() + } + } + + private var chart: some View { + let entries = self.summary.daily.compactMap { entry -> (Date, Double)? in + guard let date = CostUsageMenuDateParser.parse(entry.date) else { return nil } + return (date, entry.totalCost) + } + + return Chart(entries, id: \.0) { entry in + BarMark( + x: .value("Day", entry.0), + y: .value("Cost", entry.1)) + .foregroundStyle(Color.accentColor) + .cornerRadius(3) + } + .chartXAxis { + AxisMarks(values: .stride(by: .day, count: 7)) { + AxisGridLine().foregroundStyle(.clear) + AxisValueLabel(format: .dateTime.month().day()) + } + } + .chartYAxis { + AxisMarks(position: .leading) { + AxisGridLine() + AxisValueLabel() + } + } + .frame(height: 110) + } + + private var footer: some View { + if self.summary.totals.missingCostEntries == 0 { + return AnyView(EmptyView()) + } + return AnyView( + Text("Partial: \(self.summary.totals.missingCostEntries) entries missing cost") + .font(.caption2) + .foregroundStyle(.secondary)) + } +} + +private enum CostUsageMenuDateParser { + static let formatter: DateFormatter = { + let formatter = DateFormatter() + formatter.dateFormat = "yyyy-MM-dd" + formatter.locale = Locale(identifier: "en_US_POSIX") + formatter.timeZone = TimeZone.current + return formatter + }() + + static func parse(_ value: String) -> Date? { + self.formatter.date(from: value) + } + + static func format(_ date: Date) -> String { + self.formatter.string(from: date) + } +} diff --git a/apps/macos/Sources/OpenClaw/CritterIconRenderer.swift b/apps/macos/Sources/OpenClaw/CritterIconRenderer.swift new file mode 100644 index 0000000000000000000000000000000000000000..0309461965ceaa19b6e4594dbd77f1d4a547d919 --- /dev/null +++ b/apps/macos/Sources/OpenClaw/CritterIconRenderer.swift @@ -0,0 +1,387 @@ +import AppKit + +enum CritterIconRenderer { + private static let size = NSSize(width: 18, height: 18) + + struct Badge { + let symbolName: String + let prominence: IconState.BadgeProminence + } + + private struct Canvas { + let w: CGFloat + let h: CGFloat + let stepX: CGFloat + let stepY: CGFloat + let snapX: (CGFloat) -> CGFloat + let snapY: (CGFloat) -> CGFloat + let context: CGContext + } + + private struct Geometry { + let bodyRect: CGRect + let bodyCorner: CGFloat + let leftEarRect: CGRect + let rightEarRect: CGRect + let earCorner: CGFloat + let earW: CGFloat + let earH: CGFloat + let legW: CGFloat + let legH: CGFloat + let legSpacing: CGFloat + let legStartX: CGFloat + let legYBase: CGFloat + let legLift: CGFloat + let legHeightScale: CGFloat + let eyeW: CGFloat + let eyeY: CGFloat + let eyeOffset: CGFloat + + init(canvas: Canvas, legWiggle: CGFloat, earWiggle: CGFloat, earScale: CGFloat) { + let w = canvas.w + let h = canvas.h + let snapX = canvas.snapX + let snapY = canvas.snapY + + let bodyW = snapX(w * 0.78) + let bodyH = snapY(h * 0.58) + let bodyX = snapX((w - bodyW) / 2) + let bodyY = snapY(h * 0.36) + let bodyCorner = snapX(w * 0.09) + + let earW = snapX(w * 0.22) + let earH = snapY(bodyH * 0.54 * earScale * (1 - 0.08 * abs(earWiggle))) + let earCorner = snapX(earW * 0.24) + let leftEarRect = CGRect( + x: snapX(bodyX - earW * 0.55 + earWiggle), + y: snapY(bodyY + bodyH * 0.08 + earWiggle * 0.4), + width: earW, + height: earH) + let rightEarRect = CGRect( + x: snapX(bodyX + bodyW - earW * 0.45 - earWiggle), + y: snapY(bodyY + bodyH * 0.08 - earWiggle * 0.4), + width: earW, + height: earH) + + let legW = snapX(w * 0.11) + let legH = snapY(h * 0.26) + let legSpacing = snapX(w * 0.085) + let legsWidth = snapX(4 * legW + 3 * legSpacing) + let legStartX = snapX((w - legsWidth) / 2) + let legLift = snapY(legH * 0.35 * legWiggle) + let legYBase = snapY(bodyY - legH + h * 0.05) + let legHeightScale = 1 - 0.12 * legWiggle + + let eyeW = snapX(bodyW * 0.2) + let eyeY = snapY(bodyY + bodyH * 0.56) + let eyeOffset = snapX(bodyW * 0.24) + + self.bodyRect = CGRect(x: bodyX, y: bodyY, width: bodyW, height: bodyH) + self.bodyCorner = bodyCorner + self.leftEarRect = leftEarRect + self.rightEarRect = rightEarRect + self.earCorner = earCorner + self.earW = earW + self.earH = earH + self.legW = legW + self.legH = legH + self.legSpacing = legSpacing + self.legStartX = legStartX + self.legYBase = legYBase + self.legLift = legLift + self.legHeightScale = legHeightScale + self.eyeW = eyeW + self.eyeY = eyeY + self.eyeOffset = eyeOffset + } + } + + private struct FaceOptions { + let blink: CGFloat + let earHoles: Bool + let earScale: CGFloat + let eyesClosedLines: Bool + } + + static func makeIcon( + blink: CGFloat, + legWiggle: CGFloat = 0, + earWiggle: CGFloat = 0, + earScale: CGFloat = 1, + earHoles: Bool = false, + eyesClosedLines: Bool = false, + badge: Badge? = nil) -> NSImage + { + guard let rep = self.makeBitmapRep() else { + return NSImage(size: self.size) + } + rep.size = self.size + + NSGraphicsContext.saveGraphicsState() + defer { NSGraphicsContext.restoreGraphicsState() } + + guard let context = NSGraphicsContext(bitmapImageRep: rep) else { + return NSImage(size: self.size) + } + NSGraphicsContext.current = context + context.imageInterpolation = .none + context.cgContext.setShouldAntialias(false) + + let canvas = self.makeCanvas(for: rep, context: context) + let geometry = Geometry(canvas: canvas, legWiggle: legWiggle, earWiggle: earWiggle, earScale: earScale) + + self.drawBody(in: canvas, geometry: geometry) + let face = FaceOptions( + blink: blink, + earHoles: earHoles, + earScale: earScale, + eyesClosedLines: eyesClosedLines) + self.drawFace(in: canvas, geometry: geometry, options: face) + + if let badge { + self.drawBadge(badge, canvas: canvas) + } + + let image = NSImage(size: size) + image.addRepresentation(rep) + image.isTemplate = true + return image + } + + private static func makeBitmapRep() -> NSBitmapImageRep? { + // Force a 36×36px backing store (2× for the 18pt logical canvas) so the menu bar icon stays crisp on Retina. + let pixelsWide = 36 + let pixelsHigh = 36 + return NSBitmapImageRep( + bitmapDataPlanes: nil, + pixelsWide: pixelsWide, + pixelsHigh: pixelsHigh, + bitsPerSample: 8, + samplesPerPixel: 4, + hasAlpha: true, + isPlanar: false, + colorSpaceName: .deviceRGB, + bitmapFormat: [], + bytesPerRow: 0, + bitsPerPixel: 0) + } + + private static func makeCanvas(for rep: NSBitmapImageRep, context: NSGraphicsContext) -> Canvas { + let stepX = self.size.width / max(CGFloat(rep.pixelsWide), 1) + let stepY = self.size.height / max(CGFloat(rep.pixelsHigh), 1) + let snapX: (CGFloat) -> CGFloat = { ($0 / stepX).rounded() * stepX } + let snapY: (CGFloat) -> CGFloat = { ($0 / stepY).rounded() * stepY } + + let w = snapX(size.width) + let h = snapY(size.height) + + return Canvas( + w: w, + h: h, + stepX: stepX, + stepY: stepY, + snapX: snapX, + snapY: snapY, + context: context.cgContext) + } + + private static func drawBody(in canvas: Canvas, geometry: Geometry) { + canvas.context.setFillColor(NSColor.labelColor.cgColor) + + canvas.context.addPath(CGPath( + roundedRect: geometry.bodyRect, + cornerWidth: geometry.bodyCorner, + cornerHeight: geometry.bodyCorner, + transform: nil)) + canvas.context.addPath(CGPath( + roundedRect: geometry.leftEarRect, + cornerWidth: geometry.earCorner, + cornerHeight: geometry.earCorner, + transform: nil)) + canvas.context.addPath(CGPath( + roundedRect: geometry.rightEarRect, + cornerWidth: geometry.earCorner, + cornerHeight: geometry.earCorner, + transform: nil)) + + for i in 0..<4 { + let x = geometry.legStartX + CGFloat(i) * (geometry.legW + geometry.legSpacing) + let lift = i % 2 == 0 ? geometry.legLift : -geometry.legLift + let rect = CGRect( + x: x, + y: geometry.legYBase + lift, + width: geometry.legW, + height: geometry.legH * geometry.legHeightScale) + canvas.context.addPath(CGPath( + roundedRect: rect, + cornerWidth: geometry.legW * 0.34, + cornerHeight: geometry.legW * 0.34, + transform: nil)) + } + canvas.context.fillPath() + } + + private static func drawFace( + in canvas: Canvas, + geometry: Geometry, + options: FaceOptions) + { + canvas.context.saveGState() + canvas.context.setBlendMode(.clear) + + let leftCenter = CGPoint( + x: canvas.snapX(canvas.w / 2 - geometry.eyeOffset), + y: canvas.snapY(geometry.eyeY)) + let rightCenter = CGPoint( + x: canvas.snapX(canvas.w / 2 + geometry.eyeOffset), + y: canvas.snapY(geometry.eyeY)) + + if options.earHoles || options.earScale > 1.05 { + let holeW = canvas.snapX(geometry.earW * 0.6) + let holeH = canvas.snapY(geometry.earH * 0.46) + let holeCorner = canvas.snapX(holeW * 0.34) + let leftHoleRect = CGRect( + x: canvas.snapX(geometry.leftEarRect.midX - holeW / 2), + y: canvas.snapY(geometry.leftEarRect.midY - holeH / 2 + geometry.earH * 0.04), + width: holeW, + height: holeH) + let rightHoleRect = CGRect( + x: canvas.snapX(geometry.rightEarRect.midX - holeW / 2), + y: canvas.snapY(geometry.rightEarRect.midY - holeH / 2 + geometry.earH * 0.04), + width: holeW, + height: holeH) + + canvas.context.addPath(CGPath( + roundedRect: leftHoleRect, + cornerWidth: holeCorner, + cornerHeight: holeCorner, + transform: nil)) + canvas.context.addPath(CGPath( + roundedRect: rightHoleRect, + cornerWidth: holeCorner, + cornerHeight: holeCorner, + transform: nil)) + } + + if options.eyesClosedLines { + let lineW = canvas.snapX(geometry.eyeW * 0.95) + let lineH = canvas.snapY(max(canvas.stepY * 2, geometry.bodyRect.height * 0.06)) + let corner = canvas.snapX(lineH * 0.6) + let leftRect = CGRect( + x: canvas.snapX(leftCenter.x - lineW / 2), + y: canvas.snapY(leftCenter.y - lineH / 2), + width: lineW, + height: lineH) + let rightRect = CGRect( + x: canvas.snapX(rightCenter.x - lineW / 2), + y: canvas.snapY(rightCenter.y - lineH / 2), + width: lineW, + height: lineH) + canvas.context.addPath(CGPath( + roundedRect: leftRect, + cornerWidth: corner, + cornerHeight: corner, + transform: nil)) + canvas.context.addPath(CGPath( + roundedRect: rightRect, + cornerWidth: corner, + cornerHeight: corner, + transform: nil)) + } else { + let eyeOpen = max(0.05, 1 - options.blink) + let eyeH = canvas.snapY(geometry.bodyRect.height * 0.26 * eyeOpen) + + let left = CGMutablePath() + left.move(to: CGPoint( + x: canvas.snapX(leftCenter.x - geometry.eyeW / 2), + y: canvas.snapY(leftCenter.y - eyeH))) + left.addLine(to: CGPoint( + x: canvas.snapX(leftCenter.x + geometry.eyeW / 2), + y: canvas.snapY(leftCenter.y))) + left.addLine(to: CGPoint( + x: canvas.snapX(leftCenter.x - geometry.eyeW / 2), + y: canvas.snapY(leftCenter.y + eyeH))) + left.closeSubpath() + + let right = CGMutablePath() + right.move(to: CGPoint( + x: canvas.snapX(rightCenter.x + geometry.eyeW / 2), + y: canvas.snapY(rightCenter.y - eyeH))) + right.addLine(to: CGPoint( + x: canvas.snapX(rightCenter.x - geometry.eyeW / 2), + y: canvas.snapY(rightCenter.y))) + right.addLine(to: CGPoint( + x: canvas.snapX(rightCenter.x + geometry.eyeW / 2), + y: canvas.snapY(rightCenter.y + eyeH))) + right.closeSubpath() + + canvas.context.addPath(left) + canvas.context.addPath(right) + } + + canvas.context.fillPath() + canvas.context.restoreGState() + } + + private static func drawBadge(_ badge: Badge, canvas: Canvas) { + let strength: CGFloat = switch badge.prominence { + case .primary: 1.0 + case .secondary: 0.58 + case .overridden: 0.85 + } + + // Bigger, higher-contrast badge: + // - Increase diameter so tool activity is noticeable. + // - Draw a filled "puck", then knock out the symbol shape (transparent hole). + // This reads better in template-rendered menu bar icons than tiny monochrome glyphs. + let diameter = canvas.snapX(canvas.w * 0.52 * (0.92 + 0.08 * strength)) // ~9–10pt on an 18pt canvas + let margin = canvas.snapX(max(0.45, canvas.w * 0.03)) + let rect = CGRect( + x: canvas.snapX(canvas.w - diameter - margin), + y: canvas.snapY(margin), + width: diameter, + height: diameter) + + canvas.context.saveGState() + canvas.context.setShouldAntialias(true) + + // Clear the underlying pixels so the badge stays readable over the critter. + canvas.context.saveGState() + canvas.context.setBlendMode(.clear) + canvas.context.addEllipse(in: rect.insetBy(dx: -1.0, dy: -1.0)) + canvas.context.fillPath() + canvas.context.restoreGState() + + let fillAlpha: CGFloat = min(1.0, 0.36 + 0.24 * strength) + let strokeAlpha: CGFloat = min(1.0, 0.78 + 0.22 * strength) + + canvas.context.setFillColor(NSColor.labelColor.withAlphaComponent(fillAlpha).cgColor) + canvas.context.addEllipse(in: rect) + canvas.context.fillPath() + + canvas.context.setStrokeColor(NSColor.labelColor.withAlphaComponent(strokeAlpha).cgColor) + canvas.context.setLineWidth(max(1.25, canvas.snapX(canvas.w * 0.075))) + canvas.context.strokeEllipse(in: rect.insetBy(dx: 0.45, dy: 0.45)) + + if let base = NSImage(systemSymbolName: badge.symbolName, accessibilityDescription: nil) { + let pointSize = max(7.0, diameter * 0.82) + let config = NSImage.SymbolConfiguration(pointSize: pointSize, weight: .black) + let symbol = base.withSymbolConfiguration(config) ?? base + symbol.isTemplate = true + + let symbolRect = rect.insetBy(dx: diameter * 0.17, dy: diameter * 0.17) + canvas.context.saveGState() + canvas.context.setBlendMode(.clear) + symbol.draw( + in: symbolRect, + from: .zero, + operation: .sourceOver, + fraction: 1, + respectFlipped: true, + hints: nil) + canvas.context.restoreGState() + } + + canvas.context.restoreGState() + } +} diff --git a/apps/macos/Sources/OpenClaw/CritterStatusLabel+Behavior.swift b/apps/macos/Sources/OpenClaw/CritterStatusLabel+Behavior.swift new file mode 100644 index 0000000000000000000000000000000000000000..e1145c4e393ef67bcabd9aef06d9ce33978f37d3 --- /dev/null +++ b/apps/macos/Sources/OpenClaw/CritterStatusLabel+Behavior.swift @@ -0,0 +1,305 @@ +import AppKit +import SwiftUI + +extension CritterStatusLabel { + private var isWorkingNow: Bool { + self.iconState.isWorking || self.isWorking + } + + private var effectiveAnimationsEnabled: Bool { + self.animationsEnabled && !self.isSleeping + } + + var body: some View { + ZStack(alignment: .topTrailing) { + self.iconImage + .frame(width: 18, height: 18) + .rotationEffect(.degrees(self.wiggleAngle), anchor: .center) + .offset(x: self.wiggleOffset) + // Avoid Combine's TimerPublisher here: on macOS 26.2 we've seen crashes inside executor checks + // triggered by its callbacks. Drive periodic updates via a Swift-concurrency task instead. + .task(id: self.tickTaskID) { + guard self.effectiveAnimationsEnabled, !self.earBoostActive else { + await MainActor.run { self.resetMotion() } + return + } + + while !Task.isCancelled { + let now = Date() + await MainActor.run { self.tick(now) } + try? await Task.sleep(nanoseconds: 350_000_000) + } + } + .onChange(of: self.isPaused) { _, _ in self.resetMotion() } + .onChange(of: self.blinkTick) { _, _ in + guard self.effectiveAnimationsEnabled, !self.earBoostActive else { return } + self.blink() + } + .onChange(of: self.sendCelebrationTick) { _, _ in + guard self.effectiveAnimationsEnabled, !self.earBoostActive else { return } + self.wiggleLegs() + } + .onChange(of: self.animationsEnabled) { _, enabled in + if enabled, !self.isSleeping { + self.scheduleRandomTimers(from: Date()) + } else { + self.resetMotion() + } + } + .onChange(of: self.isSleeping) { _, _ in + self.resetMotion() + } + .onChange(of: self.earBoostActive) { _, active in + if active { + self.resetMotion() + } else if self.effectiveAnimationsEnabled { + self.scheduleRandomTimers(from: Date()) + } + } + + if self.gatewayNeedsAttention { + Circle() + .fill(self.gatewayBadgeColor) + .frame(width: 6, height: 6) + .padding(1) + } + } + .frame(width: 18, height: 18) + } + + private var tickTaskID: Int { + // Ensure SwiftUI restarts (and cancels) the task when these change. + (self.effectiveAnimationsEnabled ? 1 : 0) | (self.earBoostActive ? 2 : 0) + } + + private func tick(_ now: Date) { + guard self.effectiveAnimationsEnabled, !self.earBoostActive else { + self.resetMotion() + return + } + + if now >= self.nextBlink { + self.blink() + self.nextBlink = now.addingTimeInterval(Double.random(in: 3.5...8.5)) + } + + if now >= self.nextWiggle { + self.wiggle() + self.nextWiggle = now.addingTimeInterval(Double.random(in: 6.5...14)) + } + + if now >= self.nextLegWiggle { + self.wiggleLegs() + self.nextLegWiggle = now.addingTimeInterval(Double.random(in: 5.0...11.0)) + } + + if now >= self.nextEarWiggle { + self.wiggleEars() + self.nextEarWiggle = now.addingTimeInterval(Double.random(in: 7.0...14.0)) + } + + if self.isWorkingNow { + self.scurry() + } + } + + private var iconImage: Image { + let badge: CritterIconRenderer.Badge? = if let prominence = self.iconState.badgeProminence, !self.isPaused { + CritterIconRenderer.Badge( + symbolName: self.iconState.badgeSymbolName, + prominence: prominence) + } else { + nil + } + + if self.isPaused { + return Image(nsImage: CritterIconRenderer.makeIcon(blink: 0, badge: nil)) + } + + if self.isSleeping { + return Image(nsImage: CritterIconRenderer.makeIcon(blink: 1, eyesClosedLines: true, badge: nil)) + } + + return Image(nsImage: CritterIconRenderer.makeIcon( + blink: self.blinkAmount, + legWiggle: max(self.legWiggle, self.isWorkingNow ? 0.6 : 0), + earWiggle: self.earWiggle, + earScale: self.earBoostActive ? 1.9 : 1.0, + earHoles: self.earBoostActive, + badge: badge)) + } + + private func resetMotion() { + self.blinkAmount = 0 + self.wiggleAngle = 0 + self.wiggleOffset = 0 + self.legWiggle = 0 + self.earWiggle = 0 + } + + private func blink() { + withAnimation(.easeInOut(duration: 0.08)) { self.blinkAmount = 1 } + Task { @MainActor in + try? await Task.sleep(nanoseconds: 160_000_000) + withAnimation(.easeOut(duration: 0.12)) { self.blinkAmount = 0 } + } + } + + private func wiggle() { + let targetAngle = Double.random(in: -4.5...4.5) + let targetOffset = CGFloat.random(in: -0.5...0.5) + withAnimation(.interpolatingSpring(stiffness: 220, damping: 18)) { + self.wiggleAngle = targetAngle + self.wiggleOffset = targetOffset + } + Task { @MainActor in + try? await Task.sleep(nanoseconds: 360_000_000) + withAnimation(.interpolatingSpring(stiffness: 220, damping: 18)) { + self.wiggleAngle = 0 + self.wiggleOffset = 0 + } + } + } + + private func wiggleLegs() { + let target = CGFloat.random(in: 0.35...0.9) + withAnimation(.easeInOut(duration: 0.14)) { + self.legWiggle = target + } + Task { @MainActor in + try? await Task.sleep(nanoseconds: 220_000_000) + withAnimation(.easeOut(duration: 0.18)) { self.legWiggle = 0 } + } + } + + private func scurry() { + let target = CGFloat.random(in: 0.7...1.0) + withAnimation(.easeInOut(duration: 0.12)) { + self.legWiggle = target + self.wiggleOffset = CGFloat.random(in: -0.6...0.6) + } + Task { @MainActor in + try? await Task.sleep(nanoseconds: 180_000_000) + withAnimation(.easeOut(duration: 0.16)) { + self.legWiggle = 0.25 + self.wiggleOffset = 0 + } + } + } + + private func wiggleEars() { + let target = CGFloat.random(in: -1.2...1.2) + withAnimation(.interpolatingSpring(stiffness: 260, damping: 19)) { + self.earWiggle = target + } + Task { @MainActor in + try? await Task.sleep(nanoseconds: 320_000_000) + withAnimation(.interpolatingSpring(stiffness: 260, damping: 19)) { + self.earWiggle = 0 + } + } + } + + private func scheduleRandomTimers(from date: Date) { + self.nextBlink = date.addingTimeInterval(Double.random(in: 3.5...8.5)) + self.nextWiggle = date.addingTimeInterval(Double.random(in: 6.5...14)) + self.nextLegWiggle = date.addingTimeInterval(Double.random(in: 5.0...11.0)) + self.nextEarWiggle = date.addingTimeInterval(Double.random(in: 7.0...14.0)) + } + + private var gatewayNeedsAttention: Bool { + if self.isSleeping { return false } + switch self.gatewayStatus { + case .failed, .stopped: + return !self.isPaused + case .starting, .running, .attachedExisting: + return false + } + } + + private var gatewayBadgeColor: Color { + switch self.gatewayStatus { + case .failed: .red + case .stopped: .orange + default: .clear + } + } +} + +#if DEBUG +@MainActor +extension CritterStatusLabel { + static func exerciseForTesting() async { + var label = CritterStatusLabel( + isPaused: false, + isSleeping: false, + isWorking: true, + earBoostActive: false, + blinkTick: 1, + sendCelebrationTick: 1, + gatewayStatus: .running(details: nil), + animationsEnabled: true, + iconState: .workingMain(.tool(.bash))) + + _ = label.body + _ = label.iconImage + _ = label.tickTaskID + label.tick(Date()) + label.resetMotion() + label.blink() + label.wiggle() + label.wiggleLegs() + label.wiggleEars() + label.scurry() + label.scheduleRandomTimers(from: Date()) + _ = label.gatewayNeedsAttention + _ = label.gatewayBadgeColor + + label.isPaused = true + _ = label.iconImage + + label.isPaused = false + label.isSleeping = true + _ = label.iconImage + + label.isSleeping = false + label.iconState = .idle + _ = label.iconImage + + let failed = CritterStatusLabel( + isPaused: false, + isSleeping: false, + isWorking: false, + earBoostActive: false, + blinkTick: 0, + sendCelebrationTick: 0, + gatewayStatus: .failed("boom"), + animationsEnabled: false, + iconState: .idle) + _ = failed.gatewayNeedsAttention + _ = failed.gatewayBadgeColor + + let stopped = CritterStatusLabel( + isPaused: false, + isSleeping: false, + isWorking: false, + earBoostActive: false, + blinkTick: 0, + sendCelebrationTick: 0, + gatewayStatus: .stopped, + animationsEnabled: false, + iconState: .idle) + _ = stopped.gatewayNeedsAttention + _ = stopped.gatewayBadgeColor + + _ = CritterIconRenderer.makeIcon( + blink: 0.6, + legWiggle: 0.8, + earWiggle: 0.4, + earScale: 1.4, + earHoles: true, + eyesClosedLines: true, + badge: .init(symbolName: "gearshape.fill", prominence: .secondary)) + } +} +#endif diff --git a/apps/macos/Sources/OpenClaw/CritterStatusLabel.swift b/apps/macos/Sources/OpenClaw/CritterStatusLabel.swift new file mode 100644 index 0000000000000000000000000000000000000000..beeffdf8dd7472c57d12e4a043903e4f146e3019 --- /dev/null +++ b/apps/macos/Sources/OpenClaw/CritterStatusLabel.swift @@ -0,0 +1,23 @@ +import SwiftUI + +struct CritterStatusLabel: View { + var isPaused: Bool + var isSleeping: Bool + var isWorking: Bool + var earBoostActive: Bool + var blinkTick: Int + var sendCelebrationTick: Int + var gatewayStatus: GatewayProcessManager.Status + var animationsEnabled: Bool + var iconState: IconState + + @State var blinkAmount: CGFloat = 0 + @State var nextBlink = Date().addingTimeInterval(Double.random(in: 3.5...8.5)) + @State var wiggleAngle: Double = 0 + @State var wiggleOffset: CGFloat = 0 + @State var nextWiggle = Date().addingTimeInterval(Double.random(in: 6.5...14)) + @State var legWiggle: CGFloat = 0 + @State var nextLegWiggle = Date().addingTimeInterval(Double.random(in: 5.0...11.0)) + @State var earWiggle: CGFloat = 0 + @State var nextEarWiggle = Date().addingTimeInterval(Double.random(in: 7.0...14.0)) +} diff --git a/apps/macos/Sources/OpenClaw/CronJobEditor+Helpers.swift b/apps/macos/Sources/OpenClaw/CronJobEditor+Helpers.swift new file mode 100644 index 0000000000000000000000000000000000000000..720c8ba21caae3359037755db90290fa6a85a16b --- /dev/null +++ b/apps/macos/Sources/OpenClaw/CronJobEditor+Helpers.swift @@ -0,0 +1,260 @@ +import OpenClawProtocol +import Foundation +import SwiftUI + +extension CronJobEditor { + func gridLabel(_ text: String) -> some View { + Text(text) + .foregroundStyle(.secondary) + .frame(width: self.labelColumnWidth, alignment: .leading) + } + + func hydrateFromJob() { + guard let job else { return } + self.name = job.name + self.description = job.description ?? "" + self.agentId = job.agentId ?? "" + self.enabled = job.enabled + self.deleteAfterRun = job.deleteAfterRun ?? false + self.sessionTarget = job.sessionTarget + self.wakeMode = job.wakeMode + + switch job.schedule { + case let .at(atMs): + self.scheduleKind = .at + self.atDate = Date(timeIntervalSince1970: TimeInterval(atMs) / 1000) + case let .every(everyMs, _): + self.scheduleKind = .every + self.everyText = self.formatDuration(ms: everyMs) + case let .cron(expr, tz): + self.scheduleKind = .cron + self.cronExpr = expr + self.cronTz = tz ?? "" + } + + switch job.payload { + case let .systemEvent(text): + self.payloadKind = .systemEvent + self.systemEventText = text + case let .agentTurn(message, thinking, timeoutSeconds, deliver, channel, to, bestEffortDeliver): + self.payloadKind = .agentTurn + self.agentMessage = message + self.thinking = thinking ?? "" + self.timeoutSeconds = timeoutSeconds.map(String.init) ?? "" + self.deliver = deliver ?? false + let trimmed = (channel ?? "").trimmingCharacters(in: .whitespacesAndNewlines) + self.channel = trimmed.isEmpty ? "last" : trimmed + self.to = to ?? "" + self.bestEffortDeliver = bestEffortDeliver ?? false + } + + self.postPrefix = job.isolation?.postToMainPrefix ?? "Cron" + } + + func save() { + do { + self.error = nil + let payload = try self.buildPayload() + self.onSave(payload) + } catch { + self.error = error.localizedDescription + } + } + + func buildPayload() throws -> [String: AnyCodable] { + let name = try self.requireName() + let description = self.trimmed(self.description) + let agentId = self.trimmed(self.agentId) + let schedule = try self.buildSchedule() + let payload = try self.buildSelectedPayload() + + try self.validateSessionTarget(payload) + try self.validatePayloadRequiredFields(payload) + + var root: [String: Any] = [ + "name": name, + "enabled": self.enabled, + "schedule": schedule, + "sessionTarget": self.sessionTarget.rawValue, + "wakeMode": self.wakeMode.rawValue, + "payload": payload, + ] + self.applyDeleteAfterRun(to: &root) + if !description.isEmpty { root["description"] = description } + if !agentId.isEmpty { + root["agentId"] = agentId + } else if self.job?.agentId != nil { + root["agentId"] = NSNull() + } + + if self.sessionTarget == .isolated { + let trimmed = self.postPrefix.trimmingCharacters(in: .whitespacesAndNewlines) + root["isolation"] = [ + "postToMainPrefix": trimmed.isEmpty ? "Cron" : trimmed, + ] + } + + return root.mapValues { AnyCodable($0) } + } + + func trimmed(_ value: String) -> String { + value.trimmingCharacters(in: .whitespacesAndNewlines) + } + + func requireName() throws -> String { + let name = self.trimmed(self.name) + if name.isEmpty { + throw NSError( + domain: "Cron", + code: 0, + userInfo: [NSLocalizedDescriptionKey: "Name is required."]) + } + return name + } + + func buildSchedule() throws -> [String: Any] { + switch self.scheduleKind { + case .at: + return ["kind": "at", "atMs": Int(self.atDate.timeIntervalSince1970 * 1000)] + case .every: + guard let ms = Self.parseDurationMs(self.everyText) else { + throw NSError( + domain: "Cron", + code: 0, + userInfo: [NSLocalizedDescriptionKey: "Invalid every duration (use 10m, 1h, 1d)."]) + } + return ["kind": "every", "everyMs": ms] + case .cron: + let expr = self.trimmed(self.cronExpr) + if expr.isEmpty { + throw NSError( + domain: "Cron", + code: 0, + userInfo: [NSLocalizedDescriptionKey: "Cron expression is required."]) + } + let tz = self.trimmed(self.cronTz) + if tz.isEmpty { + return ["kind": "cron", "expr": expr] + } + return ["kind": "cron", "expr": expr, "tz": tz] + } + } + + func buildSelectedPayload() throws -> [String: Any] { + if self.sessionTarget == .isolated { return self.buildAgentTurnPayload() } + switch self.payloadKind { + case .systemEvent: + let text = self.trimmed(self.systemEventText) + return ["kind": "systemEvent", "text": text] + case .agentTurn: + return self.buildAgentTurnPayload() + } + } + + func validateSessionTarget(_ payload: [String: Any]) throws { + if self.sessionTarget == .main, payload["kind"] as? String == "agentTurn" { + throw NSError( + domain: "Cron", + code: 0, + userInfo: [ + NSLocalizedDescriptionKey: + "Main session jobs require systemEvent payloads (switch Session target to isolated).", + ]) + } + + if self.sessionTarget == .isolated, payload["kind"] as? String == "systemEvent" { + throw NSError( + domain: "Cron", + code: 0, + userInfo: [NSLocalizedDescriptionKey: "Isolated jobs require agentTurn payloads."]) + } + } + + func validatePayloadRequiredFields(_ payload: [String: Any]) throws { + if payload["kind"] as? String == "systemEvent" { + if (payload["text"] as? String ?? "").isEmpty { + throw NSError( + domain: "Cron", + code: 0, + userInfo: [NSLocalizedDescriptionKey: "System event text is required."]) + } + } + if payload["kind"] as? String == "agentTurn" { + if (payload["message"] as? String ?? "").isEmpty { + throw NSError( + domain: "Cron", + code: 0, + userInfo: [NSLocalizedDescriptionKey: "Agent message is required."]) + } + } + } + + func applyDeleteAfterRun( + to root: inout [String: Any], + scheduleKind: ScheduleKind? = nil, + deleteAfterRun: Bool? = nil) + { + let resolvedSchedule = scheduleKind ?? self.scheduleKind + let resolvedDelete = deleteAfterRun ?? self.deleteAfterRun + if resolvedSchedule == .at { + root["deleteAfterRun"] = resolvedDelete + } else if self.job?.deleteAfterRun != nil { + root["deleteAfterRun"] = false + } + } + + func buildAgentTurnPayload() -> [String: Any] { + let msg = self.agentMessage.trimmingCharacters(in: .whitespacesAndNewlines) + var payload: [String: Any] = ["kind": "agentTurn", "message": msg] + let thinking = self.thinking.trimmingCharacters(in: .whitespacesAndNewlines) + if !thinking.isEmpty { payload["thinking"] = thinking } + if let n = Int(self.timeoutSeconds), n > 0 { payload["timeoutSeconds"] = n } + payload["deliver"] = self.deliver + if self.deliver { + let trimmed = self.channel.trimmingCharacters(in: .whitespacesAndNewlines) + payload["channel"] = trimmed.isEmpty ? "last" : trimmed + let to = self.to.trimmingCharacters(in: .whitespacesAndNewlines) + if !to.isEmpty { payload["to"] = to } + payload["bestEffortDeliver"] = self.bestEffortDeliver + } + return payload + } + + static func parseDurationMs(_ input: String) -> Int? { + let raw = input.trimmingCharacters(in: .whitespacesAndNewlines) + if raw.isEmpty { return nil } + + let rx = try? NSRegularExpression(pattern: "^(\\d+(?:\\.\\d+)?)(ms|s|m|h|d)$", options: [.caseInsensitive]) + guard let match = rx?.firstMatch(in: raw, range: NSRange(location: 0, length: raw.utf16.count)) else { + return nil + } + func group(_ idx: Int) -> String { + let range = match.range(at: idx) + guard let r = Range(range, in: raw) else { return "" } + return String(raw[r]) + } + let n = Double(group(1)) ?? 0 + if !n.isFinite || n <= 0 { return nil } + let unit = group(2).lowercased() + let factor: Double = switch unit { + case "ms": 1 + case "s": 1000 + case "m": 60000 + case "h": 3_600_000 + default: 86_400_000 + } + return Int(floor(n * factor)) + } + + func formatDuration(ms: Int) -> String { + if ms < 1000 { return "\(ms)ms" } + let s = Double(ms) / 1000.0 + if s < 60 { return "\(Int(round(s)))s" } + let m = s / 60.0 + if m < 60 { return "\(Int(round(m)))m" } + let h = m / 60.0 + if h < 48 { return "\(Int(round(h)))h" } + let d = h / 24.0 + return "\(Int(round(d)))d" + } +} diff --git a/apps/macos/Sources/OpenClaw/CronJobEditor+Testing.swift b/apps/macos/Sources/OpenClaw/CronJobEditor+Testing.swift new file mode 100644 index 0000000000000000000000000000000000000000..0d4c4652369ca75b2d3c4440edec7b96786e289c --- /dev/null +++ b/apps/macos/Sources/OpenClaw/CronJobEditor+Testing.swift @@ -0,0 +1,29 @@ +#if DEBUG +extension CronJobEditor { + mutating func exerciseForTesting() { + self.name = "Test job" + self.description = "Test description" + self.agentId = "ops" + self.enabled = true + self.sessionTarget = .isolated + self.wakeMode = .now + + self.scheduleKind = .every + self.everyText = "15m" + + self.payloadKind = .agentTurn + self.agentMessage = "Run diagnostic" + self.deliver = true + self.channel = "last" + self.to = "+15551230000" + self.thinking = "low" + self.timeoutSeconds = "90" + self.bestEffortDeliver = true + self.postPrefix = "Cron" + + _ = self.buildAgentTurnPayload() + _ = try? self.buildPayload() + _ = self.formatDuration(ms: 45000) + } +} +#endif diff --git a/apps/macos/Sources/OpenClaw/CronJobEditor.swift b/apps/macos/Sources/OpenClaw/CronJobEditor.swift new file mode 100644 index 0000000000000000000000000000000000000000..6300afb5aaab310d4fba01ee5c2a5c096a19511d --- /dev/null +++ b/apps/macos/Sources/OpenClaw/CronJobEditor.swift @@ -0,0 +1,377 @@ +import OpenClawProtocol +import Observation +import SwiftUI + +struct CronJobEditor: View { + let job: CronJob? + @Binding var isSaving: Bool + @Binding var error: String? + @Bindable var channelsStore: ChannelsStore + let onCancel: () -> Void + let onSave: ([String: AnyCodable]) -> Void + + let labelColumnWidth: CGFloat = 160 + static let introText = + "Create a schedule that wakes OpenClaw via the Gateway. " + + "Use an isolated session for agent turns so your main chat stays clean." + static let sessionTargetNote = + "Main jobs post a system event into the current main session. " + + "Isolated jobs run OpenClaw in a dedicated session and can deliver results (WhatsApp/Telegram/Discord/etc)." + static let scheduleKindNote = + "“At” runs once, “Every” repeats with a duration, “Cron” uses a 5-field Unix expression." + static let isolatedPayloadNote = + "Isolated jobs always run an agent turn. The result can be delivered to a channel, " + + "and a short summary is posted back to your main chat." + static let mainPayloadNote = + "System events are injected into the current main session. Agent turns require an isolated session target." + static let mainSummaryNote = + "Controls the label used when posting the completion summary back to the main session." + + @State var name: String = "" + @State var description: String = "" + @State var agentId: String = "" + @State var enabled: Bool = true + @State var sessionTarget: CronSessionTarget = .main + @State var wakeMode: CronWakeMode = .nextHeartbeat + @State var deleteAfterRun: Bool = false + + enum ScheduleKind: String, CaseIterable, Identifiable { case at, every, cron; var id: String { rawValue } } + @State var scheduleKind: ScheduleKind = .every + @State var atDate: Date = .init().addingTimeInterval(60 * 5) + @State var everyText: String = "1h" + @State var cronExpr: String = "0 9 * * 3" + @State var cronTz: String = "" + + enum PayloadKind: String, CaseIterable, Identifiable { case systemEvent, agentTurn; var id: String { rawValue } } + @State var payloadKind: PayloadKind = .systemEvent + @State var systemEventText: String = "" + @State var agentMessage: String = "" + @State var deliver: Bool = false + @State var channel: String = "last" + @State var to: String = "" + @State var thinking: String = "" + @State var timeoutSeconds: String = "" + @State var bestEffortDeliver: Bool = false + @State var postPrefix: String = "Cron" + + var channelOptions: [String] { + let ordered = self.channelsStore.orderedChannelIds() + var options = ["last"] + ordered + let trimmed = self.channel.trimmingCharacters(in: .whitespacesAndNewlines) + if !trimmed.isEmpty, !options.contains(trimmed) { + options.append(trimmed) + } + var seen = Set() + return options.filter { seen.insert($0).inserted } + } + + func channelLabel(for id: String) -> String { + if id == "last" { return "last" } + return self.channelsStore.resolveChannelLabel(id) + } + + var body: some View { + VStack(alignment: .leading, spacing: 16) { + VStack(alignment: .leading, spacing: 6) { + Text(self.job == nil ? "New cron job" : "Edit cron job") + .font(.title3.weight(.semibold)) + Text(Self.introText) + .font(.callout) + .foregroundStyle(.secondary) + .fixedSize(horizontal: false, vertical: true) + } + + ScrollView(.vertical) { + VStack(alignment: .leading, spacing: 14) { + GroupBox("Basics") { + Grid(alignment: .leadingFirstTextBaseline, horizontalSpacing: 14, verticalSpacing: 10) { + GridRow { + self.gridLabel("Name") + TextField("Required (e.g. “Daily summary”)", text: self.$name) + .textFieldStyle(.roundedBorder) + .frame(maxWidth: .infinity) + } + GridRow { + self.gridLabel("Description") + TextField("Optional notes", text: self.$description) + .textFieldStyle(.roundedBorder) + .frame(maxWidth: .infinity) + } + GridRow { + self.gridLabel("Agent ID") + TextField("Optional (default agent)", text: self.$agentId) + .textFieldStyle(.roundedBorder) + .frame(maxWidth: .infinity) + } + GridRow { + self.gridLabel("Enabled") + Toggle("", isOn: self.$enabled) + .labelsHidden() + .toggleStyle(.switch) + } + GridRow { + self.gridLabel("Session target") + Picker("", selection: self.$sessionTarget) { + Text("main").tag(CronSessionTarget.main) + Text("isolated").tag(CronSessionTarget.isolated) + } + .labelsHidden() + .pickerStyle(.segmented) + .frame(maxWidth: .infinity, alignment: .leading) + } + GridRow { + self.gridLabel("Wake mode") + Picker("", selection: self.$wakeMode) { + Text("next-heartbeat").tag(CronWakeMode.nextHeartbeat) + Text("now").tag(CronWakeMode.now) + } + .labelsHidden() + .pickerStyle(.segmented) + .frame(maxWidth: .infinity, alignment: .leading) + } + GridRow { + Color.clear + .frame(width: self.labelColumnWidth, height: 1) + Text( + Self.sessionTargetNote) + .font(.footnote) + .foregroundStyle(.secondary) + .frame(maxWidth: .infinity, alignment: .leading) + } + } + } + + GroupBox("Schedule") { + Grid(alignment: .leadingFirstTextBaseline, horizontalSpacing: 14, verticalSpacing: 10) { + GridRow { + self.gridLabel("Kind") + Picker("", selection: self.$scheduleKind) { + Text("at").tag(ScheduleKind.at) + Text("every").tag(ScheduleKind.every) + Text("cron").tag(ScheduleKind.cron) + } + .labelsHidden() + .pickerStyle(.segmented) + .frame(maxWidth: .infinity) + } + GridRow { + Color.clear + .frame(width: self.labelColumnWidth, height: 1) + Text( + Self.scheduleKindNote) + .font(.footnote) + .foregroundStyle(.secondary) + .frame(maxWidth: .infinity, alignment: .leading) + } + + switch self.scheduleKind { + case .at: + GridRow { + self.gridLabel("At") + DatePicker( + "", + selection: self.$atDate, + displayedComponents: [.date, .hourAndMinute]) + .labelsHidden() + .frame(maxWidth: .infinity, alignment: .leading) + } + GridRow { + self.gridLabel("Auto-delete") + Toggle("Delete after successful run", isOn: self.$deleteAfterRun) + .toggleStyle(.switch) + } + case .every: + GridRow { + self.gridLabel("Every") + TextField("10m, 1h, 1d", text: self.$everyText) + .textFieldStyle(.roundedBorder) + .frame(maxWidth: .infinity) + } + case .cron: + GridRow { + self.gridLabel("Expression") + TextField("e.g. 0 9 * * 3", text: self.$cronExpr) + .textFieldStyle(.roundedBorder) + .frame(maxWidth: .infinity) + } + GridRow { + self.gridLabel("Timezone") + TextField("Optional (e.g. America/Los_Angeles)", text: self.$cronTz) + .textFieldStyle(.roundedBorder) + .frame(maxWidth: .infinity) + } + } + } + } + + GroupBox("Payload") { + VStack(alignment: .leading, spacing: 10) { + if self.sessionTarget == .isolated { + Text(Self.isolatedPayloadNote) + .font(.footnote) + .foregroundStyle(.secondary) + .fixedSize(horizontal: false, vertical: true) + self.agentTurnEditor + } else { + Grid(alignment: .leadingFirstTextBaseline, horizontalSpacing: 14, verticalSpacing: 10) { + GridRow { + self.gridLabel("Kind") + Picker("", selection: self.$payloadKind) { + Text("systemEvent").tag(PayloadKind.systemEvent) + Text("agentTurn").tag(PayloadKind.agentTurn) + } + .labelsHidden() + .pickerStyle(.segmented) + .frame(maxWidth: .infinity) + } + GridRow { + Color.clear + .frame(width: self.labelColumnWidth, height: 1) + Text( + Self.mainPayloadNote) + .font(.footnote) + .foregroundStyle(.secondary) + .frame(maxWidth: .infinity, alignment: .leading) + } + } + + switch self.payloadKind { + case .systemEvent: + TextField("System event text", text: self.$systemEventText, axis: .vertical) + .textFieldStyle(.roundedBorder) + .lineLimit(3...7) + .frame(maxWidth: .infinity) + case .agentTurn: + self.agentTurnEditor + } + } + } + } + + if self.sessionTarget == .isolated { + GroupBox("Main session summary") { + Grid(alignment: .leadingFirstTextBaseline, horizontalSpacing: 14, verticalSpacing: 10) { + GridRow { + self.gridLabel("Prefix") + TextField("Cron", text: self.$postPrefix) + .textFieldStyle(.roundedBorder) + .frame(maxWidth: .infinity) + } + GridRow { + Color.clear + .frame(width: self.labelColumnWidth, height: 1) + Text( + Self.mainSummaryNote) + .font(.footnote) + .foregroundStyle(.secondary) + .frame(maxWidth: .infinity, alignment: .leading) + } + } + } + } + } + .frame(maxWidth: .infinity, alignment: .leading) + .padding(.vertical, 2) + } + + if let error, !error.isEmpty { + Text(error) + .font(.footnote) + .foregroundStyle(.red) + .fixedSize(horizontal: false, vertical: true) + } + + HStack { + Button("Cancel") { self.onCancel() } + .keyboardShortcut(.cancelAction) + .buttonStyle(.bordered) + Spacer() + Button { + self.save() + } label: { + if self.isSaving { + ProgressView().controlSize(.small) + } else { + Text("Save") + } + } + .keyboardShortcut(.defaultAction) + .buttonStyle(.borderedProminent) + .disabled(self.isSaving) + } + } + .padding(24) + .frame(minWidth: 720, minHeight: 640) + .onAppear { self.hydrateFromJob() } + .onChange(of: self.payloadKind) { _, newValue in + if newValue == .agentTurn, self.sessionTarget == .main { + self.sessionTarget = .isolated + } + } + .onChange(of: self.sessionTarget) { _, newValue in + if newValue == .isolated { + self.payloadKind = .agentTurn + } else if newValue == .main, self.payloadKind == .agentTurn { + self.payloadKind = .systemEvent + } + } + } + + var agentTurnEditor: some View { + VStack(alignment: .leading, spacing: 10) { + Grid(alignment: .leadingFirstTextBaseline, horizontalSpacing: 14, verticalSpacing: 10) { + GridRow { + self.gridLabel("Message") + TextField("What should OpenClaw do?", text: self.$agentMessage, axis: .vertical) + .textFieldStyle(.roundedBorder) + .lineLimit(3...7) + .frame(maxWidth: .infinity) + } + GridRow { + self.gridLabel("Thinking") + TextField("Optional (e.g. low)", text: self.$thinking) + .textFieldStyle(.roundedBorder) + .frame(maxWidth: .infinity) + } + GridRow { + self.gridLabel("Timeout") + TextField("Seconds (optional)", text: self.$timeoutSeconds) + .textFieldStyle(.roundedBorder) + .frame(width: 180, alignment: .leading) + } + GridRow { + self.gridLabel("Deliver") + Toggle("Deliver result to a channel", isOn: self.$deliver) + .toggleStyle(.switch) + } + } + + if self.deliver { + Grid(alignment: .leadingFirstTextBaseline, horizontalSpacing: 14, verticalSpacing: 10) { + GridRow { + self.gridLabel("Channel") + Picker("", selection: self.$channel) { + ForEach(self.channelOptions, id: \.self) { channel in + Text(self.channelLabel(for: channel)).tag(channel) + } + } + .labelsHidden() + .pickerStyle(.segmented) + .frame(maxWidth: .infinity, alignment: .leading) + } + GridRow { + self.gridLabel("To") + TextField("Optional override (phone number / chat id / Discord channel)", text: self.$to) + .textFieldStyle(.roundedBorder) + .frame(maxWidth: .infinity) + } + GridRow { + self.gridLabel("Best-effort") + Toggle("Do not fail the job if delivery fails", isOn: self.$bestEffortDeliver) + .toggleStyle(.switch) + } + } + } + } + } +} diff --git a/apps/macos/Sources/OpenClaw/CronJobsStore.swift b/apps/macos/Sources/OpenClaw/CronJobsStore.swift new file mode 100644 index 0000000000000000000000000000000000000000..cb84a2b41fd053f1739e0c718e7897cb66120acb --- /dev/null +++ b/apps/macos/Sources/OpenClaw/CronJobsStore.swift @@ -0,0 +1,200 @@ +import OpenClawKit +import OpenClawProtocol +import Foundation +import Observation +import OSLog + +@MainActor +@Observable +final class CronJobsStore { + static let shared = CronJobsStore() + + var jobs: [CronJob] = [] + var selectedJobId: String? + var runEntries: [CronRunLogEntry] = [] + + var schedulerEnabled: Bool? + var schedulerStorePath: String? + var schedulerNextWakeAtMs: Int? + + var isLoadingJobs = false + var isLoadingRuns = false + var lastError: String? + var statusMessage: String? + + private let logger = Logger(subsystem: "ai.openclaw", category: "cron.ui") + private var refreshTask: Task? + private var runsTask: Task? + private var eventTask: Task? + private var pollTask: Task? + + private let interval: TimeInterval = 30 + private let isPreview: Bool + + init(isPreview: Bool = ProcessInfo.processInfo.isPreview) { + self.isPreview = isPreview + } + + func start() { + guard !self.isPreview else { return } + guard self.eventTask == nil else { return } + self.startGatewaySubscription() + self.pollTask = Task.detached { [weak self] in + guard let self else { return } + await self.refreshJobs() + while !Task.isCancelled { + try? await Task.sleep(nanoseconds: UInt64(self.interval * 1_000_000_000)) + await self.refreshJobs() + } + } + } + + func stop() { + self.refreshTask?.cancel() + self.refreshTask = nil + self.runsTask?.cancel() + self.runsTask = nil + self.eventTask?.cancel() + self.eventTask = nil + self.pollTask?.cancel() + self.pollTask = nil + } + + func refreshJobs() async { + guard !self.isLoadingJobs else { return } + self.isLoadingJobs = true + self.lastError = nil + self.statusMessage = nil + defer { self.isLoadingJobs = false } + + do { + if let status = try? await GatewayConnection.shared.cronStatus() { + self.schedulerEnabled = status.enabled + self.schedulerStorePath = status.storePath + self.schedulerNextWakeAtMs = status.nextWakeAtMs + } + self.jobs = try await GatewayConnection.shared.cronList(includeDisabled: true) + if self.jobs.isEmpty { + self.statusMessage = "No cron jobs yet." + } + } catch { + self.logger.error("cron.list failed \(error.localizedDescription, privacy: .public)") + self.lastError = error.localizedDescription + } + } + + func refreshRuns(jobId: String, limit: Int = 200) async { + guard !self.isLoadingRuns else { return } + self.isLoadingRuns = true + defer { self.isLoadingRuns = false } + + do { + self.runEntries = try await GatewayConnection.shared.cronRuns(jobId: jobId, limit: limit) + } catch { + self.logger.error("cron.runs failed \(error.localizedDescription, privacy: .public)") + self.lastError = error.localizedDescription + } + } + + func runJob(id: String, force: Bool = true) async { + do { + try await GatewayConnection.shared.cronRun(jobId: id, force: force) + } catch { + self.lastError = error.localizedDescription + } + } + + func removeJob(id: String) async { + do { + try await GatewayConnection.shared.cronRemove(jobId: id) + await self.refreshJobs() + if self.selectedJobId == id { + self.selectedJobId = nil + self.runEntries = [] + } + } catch { + self.lastError = error.localizedDescription + } + } + + func setJobEnabled(id: String, enabled: Bool) async { + do { + try await GatewayConnection.shared.cronUpdate( + jobId: id, + patch: ["enabled": AnyCodable(enabled)]) + await self.refreshJobs() + } catch { + self.lastError = error.localizedDescription + } + } + + func upsertJob( + id: String?, + payload: [String: AnyCodable]) async throws + { + if let id { + try await GatewayConnection.shared.cronUpdate(jobId: id, patch: payload) + } else { + try await GatewayConnection.shared.cronAdd(payload: payload) + } + await self.refreshJobs() + } + + // MARK: - Gateway events + + private func startGatewaySubscription() { + self.eventTask?.cancel() + self.eventTask = Task { [weak self] in + guard let self else { return } + let stream = await GatewayConnection.shared.subscribe() + for await push in stream { + if Task.isCancelled { return } + await MainActor.run { [weak self] in + self?.handle(push: push) + } + } + } + } + + private func handle(push: GatewayPush) { + switch push { + case let .event(evt) where evt.event == "cron": + guard let payload = evt.payload else { return } + if let cronEvt = try? GatewayPayloadDecoding.decode(payload, as: CronEvent.self) { + self.handle(cronEvent: cronEvt) + } + case .seqGap: + self.scheduleRefresh() + default: + break + } + } + + private func handle(cronEvent evt: CronEvent) { + // Keep UI in sync with the gateway scheduler. + self.scheduleRefresh(delayMs: 250) + if evt.action == "finished", let selected = self.selectedJobId, selected == evt.jobId { + self.scheduleRunsRefresh(jobId: selected, delayMs: 200) + } + } + + private func scheduleRefresh(delayMs: Int = 250) { + self.refreshTask?.cancel() + self.refreshTask = Task { [weak self] in + guard let self else { return } + try? await Task.sleep(nanoseconds: UInt64(delayMs) * 1_000_000) + await self.refreshJobs() + } + } + + private func scheduleRunsRefresh(jobId: String, delayMs: Int = 200) { + self.runsTask?.cancel() + self.runsTask = Task { [weak self] in + guard let self else { return } + try? await Task.sleep(nanoseconds: UInt64(delayMs) * 1_000_000) + await self.refreshRuns(jobId: jobId) + } + } + + // MARK: - (no additional RPC helpers) +} diff --git a/apps/macos/Sources/OpenClaw/CronModels.swift b/apps/macos/Sources/OpenClaw/CronModels.swift new file mode 100644 index 0000000000000000000000000000000000000000..7c7e77e928bfda895f95e93714fee88fc5b47602 --- /dev/null +++ b/apps/macos/Sources/OpenClaw/CronModels.swift @@ -0,0 +1,216 @@ +import Foundation + +enum CronSessionTarget: String, CaseIterable, Identifiable, Codable { + case main + case isolated + + var id: String { self.rawValue } +} + +enum CronWakeMode: String, CaseIterable, Identifiable, Codable { + case now + case nextHeartbeat = "next-heartbeat" + + var id: String { self.rawValue } +} + +enum CronSchedule: Codable, Equatable { + case at(atMs: Int) + case every(everyMs: Int, anchorMs: Int?) + case cron(expr: String, tz: String?) + + enum CodingKeys: String, CodingKey { case kind, atMs, everyMs, anchorMs, expr, tz } + + var kind: String { + switch self { + case .at: "at" + case .every: "every" + case .cron: "cron" + } + } + + init(from decoder: Decoder) throws { + let container = try decoder.container(keyedBy: CodingKeys.self) + let kind = try container.decode(String.self, forKey: .kind) + switch kind { + case "at": + self = try .at(atMs: container.decode(Int.self, forKey: .atMs)) + case "every": + self = try .every( + everyMs: container.decode(Int.self, forKey: .everyMs), + anchorMs: container.decodeIfPresent(Int.self, forKey: .anchorMs)) + case "cron": + self = try .cron( + expr: container.decode(String.self, forKey: .expr), + tz: container.decodeIfPresent(String.self, forKey: .tz)) + default: + throw DecodingError.dataCorruptedError( + forKey: .kind, + in: container, + debugDescription: "Unknown schedule kind: \(kind)") + } + } + + func encode(to encoder: Encoder) throws { + var container = encoder.container(keyedBy: CodingKeys.self) + try container.encode(self.kind, forKey: .kind) + switch self { + case let .at(atMs): + try container.encode(atMs, forKey: .atMs) + case let .every(everyMs, anchorMs): + try container.encode(everyMs, forKey: .everyMs) + try container.encodeIfPresent(anchorMs, forKey: .anchorMs) + case let .cron(expr, tz): + try container.encode(expr, forKey: .expr) + try container.encodeIfPresent(tz, forKey: .tz) + } + } +} + +enum CronPayload: Codable, Equatable { + case systemEvent(text: String) + case agentTurn( + message: String, + thinking: String?, + timeoutSeconds: Int?, + deliver: Bool?, + channel: String?, + to: String?, + bestEffortDeliver: Bool?) + + enum CodingKeys: String, CodingKey { + case kind, text, message, thinking, timeoutSeconds, deliver, channel, provider, to, bestEffortDeliver + } + + var kind: String { + switch self { + case .systemEvent: "systemEvent" + case .agentTurn: "agentTurn" + } + } + + init(from decoder: Decoder) throws { + let container = try decoder.container(keyedBy: CodingKeys.self) + let kind = try container.decode(String.self, forKey: .kind) + switch kind { + case "systemEvent": + self = try .systemEvent(text: container.decode(String.self, forKey: .text)) + case "agentTurn": + self = try .agentTurn( + message: container.decode(String.self, forKey: .message), + thinking: container.decodeIfPresent(String.self, forKey: .thinking), + timeoutSeconds: container.decodeIfPresent(Int.self, forKey: .timeoutSeconds), + deliver: container.decodeIfPresent(Bool.self, forKey: .deliver), + channel: container.decodeIfPresent(String.self, forKey: .channel) + ?? container.decodeIfPresent(String.self, forKey: .provider), + to: container.decodeIfPresent(String.self, forKey: .to), + bestEffortDeliver: container.decodeIfPresent(Bool.self, forKey: .bestEffortDeliver)) + default: + throw DecodingError.dataCorruptedError( + forKey: .kind, + in: container, + debugDescription: "Unknown payload kind: \(kind)") + } + } + + func encode(to encoder: Encoder) throws { + var container = encoder.container(keyedBy: CodingKeys.self) + try container.encode(self.kind, forKey: .kind) + switch self { + case let .systemEvent(text): + try container.encode(text, forKey: .text) + case let .agentTurn(message, thinking, timeoutSeconds, deliver, channel, to, bestEffortDeliver): + try container.encode(message, forKey: .message) + try container.encodeIfPresent(thinking, forKey: .thinking) + try container.encodeIfPresent(timeoutSeconds, forKey: .timeoutSeconds) + try container.encodeIfPresent(deliver, forKey: .deliver) + try container.encodeIfPresent(channel, forKey: .channel) + try container.encodeIfPresent(to, forKey: .to) + try container.encodeIfPresent(bestEffortDeliver, forKey: .bestEffortDeliver) + } + } +} + +struct CronIsolation: Codable, Equatable { + var postToMainPrefix: String? +} + +struct CronJobState: Codable, Equatable { + var nextRunAtMs: Int? + var runningAtMs: Int? + var lastRunAtMs: Int? + var lastStatus: String? + var lastError: String? + var lastDurationMs: Int? +} + +struct CronJob: Identifiable, Codable, Equatable { + let id: String + let agentId: String? + var name: String + var description: String? + var enabled: Bool + var deleteAfterRun: Bool? + let createdAtMs: Int + let updatedAtMs: Int + let schedule: CronSchedule + let sessionTarget: CronSessionTarget + let wakeMode: CronWakeMode + let payload: CronPayload + let isolation: CronIsolation? + let state: CronJobState + + var displayName: String { + let trimmed = self.name.trimmingCharacters(in: .whitespacesAndNewlines) + return trimmed.isEmpty ? "Untitled job" : trimmed + } + + var nextRunDate: Date? { + guard let ms = self.state.nextRunAtMs else { return nil } + return Date(timeIntervalSince1970: TimeInterval(ms) / 1000) + } + + var lastRunDate: Date? { + guard let ms = self.state.lastRunAtMs else { return nil } + return Date(timeIntervalSince1970: TimeInterval(ms) / 1000) + } +} + +struct CronEvent: Codable, Sendable { + let jobId: String + let action: String + let runAtMs: Int? + let durationMs: Int? + let status: String? + let error: String? + let summary: String? + let nextRunAtMs: Int? +} + +struct CronRunLogEntry: Codable, Identifiable, Sendable { + var id: String { "\(self.jobId)-\(self.ts)" } + + let ts: Int + let jobId: String + let action: String + let status: String? + let error: String? + let summary: String? + let runAtMs: Int? + let durationMs: Int? + let nextRunAtMs: Int? + + var date: Date { Date(timeIntervalSince1970: TimeInterval(self.ts) / 1000) } + var runDate: Date? { + guard let runAtMs else { return nil } + return Date(timeIntervalSince1970: TimeInterval(runAtMs) / 1000) + } +} + +struct CronListResponse: Codable { + let jobs: [CronJob] +} + +struct CronRunsResponse: Codable { + let entries: [CronRunLogEntry] +} diff --git a/apps/macos/Sources/OpenClaw/CronSettings+Actions.swift b/apps/macos/Sources/OpenClaw/CronSettings+Actions.swift new file mode 100644 index 0000000000000000000000000000000000000000..d5fe92ae01007b00020a21b4490f0ce12e3d4c40 --- /dev/null +++ b/apps/macos/Sources/OpenClaw/CronSettings+Actions.swift @@ -0,0 +1,23 @@ +import OpenClawProtocol +import Foundation + +extension CronSettings { + func save(payload: [String: AnyCodable]) async { + guard !self.isSaving else { return } + self.isSaving = true + self.editorError = nil + do { + try await self.store.upsertJob(id: self.editingJob?.id, payload: payload) + await MainActor.run { + self.isSaving = false + self.showEditor = false + self.editingJob = nil + } + } catch { + await MainActor.run { + self.isSaving = false + self.editorError = error.localizedDescription + } + } + } +} diff --git a/apps/macos/Sources/OpenClaw/CronSettings+Helpers.swift b/apps/macos/Sources/OpenClaw/CronSettings+Helpers.swift new file mode 100644 index 0000000000000000000000000000000000000000..86f313ae59dd2e022d9b7ce1523a78005cb85970 --- /dev/null +++ b/apps/macos/Sources/OpenClaw/CronSettings+Helpers.swift @@ -0,0 +1,54 @@ +import SwiftUI + +extension CronSettings { + var selectedJob: CronJob? { + guard let id = self.store.selectedJobId else { return nil } + return self.store.jobs.first(where: { $0.id == id }) + } + + func statusTint(_ status: String?) -> Color { + switch (status ?? "").lowercased() { + case "ok": .green + case "error": .red + case "skipped": .orange + default: .secondary + } + } + + func scheduleSummary(_ schedule: CronSchedule) -> String { + switch schedule { + case let .at(atMs): + let date = Date(timeIntervalSince1970: TimeInterval(atMs) / 1000) + return "at \(date.formatted(date: .abbreviated, time: .standard))" + case let .every(everyMs, _): + return "every \(self.formatDuration(ms: everyMs))" + case let .cron(expr, tz): + if let tz, !tz.isEmpty { return "cron \(expr) (\(tz))" } + return "cron \(expr)" + } + } + + func formatDuration(ms: Int) -> String { + if ms < 1000 { return "\(ms)ms" } + let s = Double(ms) / 1000.0 + if s < 60 { return "\(Int(round(s)))s" } + let m = s / 60.0 + if m < 60 { return "\(Int(round(m)))m" } + let h = m / 60.0 + if h < 48 { return "\(Int(round(h)))h" } + let d = h / 24.0 + return "\(Int(round(d)))d" + } + + func nextRunLabel(_ date: Date, now: Date = .init()) -> String { + let delta = date.timeIntervalSince(now) + if delta <= 0 { return "due" } + if delta < 60 { return "in <1m" } + let minutes = Int(round(delta / 60)) + if minutes < 60 { return "in \(minutes)m" } + let hours = Int(round(Double(minutes) / 60)) + if hours < 48 { return "in \(hours)h" } + let days = Int(round(Double(hours) / 24)) + return "in \(days)d" + } +} diff --git a/apps/macos/Sources/OpenClaw/CronSettings+Layout.swift b/apps/macos/Sources/OpenClaw/CronSettings+Layout.swift new file mode 100644 index 0000000000000000000000000000000000000000..11c7c0a0e5be407589956cd1bd091ab2d2c5b647 --- /dev/null +++ b/apps/macos/Sources/OpenClaw/CronSettings+Layout.swift @@ -0,0 +1,179 @@ +import SwiftUI + +extension CronSettings { + var body: some View { + VStack(alignment: .leading, spacing: 12) { + self.header + self.schedulerBanner + self.content + Spacer(minLength: 0) + } + .onAppear { + self.store.start() + self.channelsStore.start() + } + .onDisappear { + self.store.stop() + self.channelsStore.stop() + } + .sheet(isPresented: self.$showEditor) { + CronJobEditor( + job: self.editingJob, + isSaving: self.$isSaving, + error: self.$editorError, + channelsStore: self.channelsStore, + onCancel: { + self.showEditor = false + self.editingJob = nil + }, + onSave: { payload in + Task { + await self.save(payload: payload) + } + }) + } + .alert("Delete cron job?", isPresented: Binding( + get: { self.confirmDelete != nil }, + set: { if !$0 { self.confirmDelete = nil } })) + { + Button("Cancel", role: .cancel) { self.confirmDelete = nil } + Button("Delete", role: .destructive) { + if let job = self.confirmDelete { + Task { await self.store.removeJob(id: job.id) } + } + self.confirmDelete = nil + } + } message: { + if let job = self.confirmDelete { + Text(job.displayName) + } + } + .onChange(of: self.store.selectedJobId) { _, newValue in + guard let newValue else { return } + Task { await self.store.refreshRuns(jobId: newValue) } + } + } + + var schedulerBanner: some View { + Group { + if self.store.schedulerEnabled == false { + VStack(alignment: .leading, spacing: 6) { + HStack(spacing: 8) { + Image(systemName: "exclamationmark.triangle.fill") + .foregroundStyle(.orange) + Text("Cron scheduler is disabled") + .font(.headline) + Spacer() + } + Text( + "Jobs are saved, but they will not run automatically until `cron.enabled` is set to `true` " + + "and the Gateway restarts.") + .font(.footnote) + .foregroundStyle(.secondary) + .fixedSize(horizontal: false, vertical: true) + if let storePath = self.store.schedulerStorePath, !storePath.isEmpty { + Text(storePath) + .font(.caption.monospaced()) + .foregroundStyle(.secondary) + .textSelection(.enabled) + .lineLimit(1) + .truncationMode(.middle) + } + } + .frame(maxWidth: .infinity, alignment: .leading) + .padding(10) + .background(Color.orange.opacity(0.10)) + .cornerRadius(8) + } + } + } + + var header: some View { + HStack(alignment: .top) { + VStack(alignment: .leading, spacing: 4) { + Text("Cron") + .font(.headline) + Text("Manage Gateway cron jobs (main session vs isolated runs) and inspect run history.") + .font(.footnote) + .foregroundStyle(.secondary) + .fixedSize(horizontal: false, vertical: true) + } + Spacer() + HStack(spacing: 8) { + Button { + Task { await self.store.refreshJobs() } + } label: { + Label("Refresh", systemImage: "arrow.clockwise") + } + .buttonStyle(.bordered) + .disabled(self.store.isLoadingJobs) + + Button { + self.editorError = nil + self.editingJob = nil + self.showEditor = true + } label: { + Label("New Job", systemImage: "plus") + } + .buttonStyle(.borderedProminent) + } + } + } + + var content: some View { + HStack(spacing: 12) { + VStack(alignment: .leading, spacing: 8) { + if let err = self.store.lastError { + Text("Error: \(err)") + .font(.footnote) + .foregroundStyle(.red) + } else if let msg = self.store.statusMessage { + Text(msg) + .font(.footnote) + .foregroundStyle(.secondary) + } + + List(selection: self.$store.selectedJobId) { + ForEach(self.store.jobs) { job in + self.jobRow(job) + .tag(job.id) + .contextMenu { self.jobContextMenu(job) } + } + } + .listStyle(.inset) + } + .frame(width: 250) + + Divider() + + self.detail + .frame(maxWidth: .infinity, maxHeight: .infinity, alignment: .topLeading) + } + } + + @ViewBuilder + var detail: some View { + if let selected = self.selectedJob { + ScrollView(.vertical) { + VStack(alignment: .leading, spacing: 12) { + self.detailHeader(selected) + self.detailCard(selected) + self.runHistoryCard(selected) + } + .frame(maxWidth: .infinity, alignment: .leading) + .padding(.top, 2) + } + } else { + VStack(alignment: .leading, spacing: 8) { + Text("Select a job to inspect details and run history.") + .font(.callout) + .foregroundStyle(.secondary) + Text("Tip: use ‘New Job’ to add one, or enable cron in your gateway config.") + .font(.caption) + .foregroundStyle(.tertiary) + } + .frame(maxWidth: .infinity, maxHeight: .infinity, alignment: .topLeading) + .padding(.top, 8) + } + } +} diff --git a/apps/macos/Sources/OpenClaw/CronSettings+Rows.swift b/apps/macos/Sources/OpenClaw/CronSettings+Rows.swift new file mode 100644 index 0000000000000000000000000000000000000000..98ebc23e6b9c2682de06260c3f9f3b0a242629a6 --- /dev/null +++ b/apps/macos/Sources/OpenClaw/CronSettings+Rows.swift @@ -0,0 +1,236 @@ +import SwiftUI + +extension CronSettings { + func jobRow(_ job: CronJob) -> some View { + VStack(alignment: .leading, spacing: 6) { + HStack(spacing: 8) { + Text(job.displayName) + .font(.subheadline.weight(.semibold)) + .lineLimit(1) + .truncationMode(.middle) + Spacer() + if !job.enabled { + StatusPill(text: "disabled", tint: .secondary) + } else if let next = job.nextRunDate { + StatusPill(text: self.nextRunLabel(next), tint: .secondary) + } else { + StatusPill(text: "no next run", tint: .secondary) + } + } + HStack(spacing: 6) { + StatusPill(text: job.sessionTarget.rawValue, tint: .secondary) + StatusPill(text: job.wakeMode.rawValue, tint: .secondary) + if let agentId = job.agentId, !agentId.isEmpty { + StatusPill(text: "agent \(agentId)", tint: .secondary) + } + if let status = job.state.lastStatus { + StatusPill(text: status, tint: status == "ok" ? .green : .orange) + } + } + } + .padding(.vertical, 6) + } + + @ViewBuilder + func jobContextMenu(_ job: CronJob) -> some View { + Button("Run now") { Task { await self.store.runJob(id: job.id, force: true) } } + if job.sessionTarget == .isolated { + Button("Open transcript") { + WebChatManager.shared.show(sessionKey: "cron:\(job.id)") + } + } + Divider() + Button(job.enabled ? "Disable" : "Enable") { + Task { await self.store.setJobEnabled(id: job.id, enabled: !job.enabled) } + } + Button("Edit…") { + self.editingJob = job + self.editorError = nil + self.showEditor = true + } + Divider() + Button("Delete…", role: .destructive) { + self.confirmDelete = job + } + } + + func detailHeader(_ job: CronJob) -> some View { + HStack(alignment: .center) { + VStack(alignment: .leading, spacing: 4) { + Text(job.displayName) + .font(.title3.weight(.semibold)) + Text(job.id) + .font(.caption.monospaced()) + .foregroundStyle(.secondary) + .textSelection(.enabled) + .lineLimit(1) + .truncationMode(.middle) + } + Spacer() + HStack(spacing: 8) { + Toggle("Enabled", isOn: Binding( + get: { job.enabled }, + set: { enabled in Task { await self.store.setJobEnabled(id: job.id, enabled: enabled) } })) + .toggleStyle(.switch) + .labelsHidden() + Button("Run") { Task { await self.store.runJob(id: job.id, force: true) } } + .buttonStyle(.borderedProminent) + if job.sessionTarget == .isolated { + Button("Transcript") { + WebChatManager.shared.show(sessionKey: "cron:\(job.id)") + } + .buttonStyle(.bordered) + } + Button("Edit") { + self.editingJob = job + self.editorError = nil + self.showEditor = true + } + .buttonStyle(.bordered) + } + } + } + + func detailCard(_ job: CronJob) -> some View { + VStack(alignment: .leading, spacing: 10) { + LabeledContent("Schedule") { Text(self.scheduleSummary(job.schedule)).font(.callout) } + if case .at = job.schedule, job.deleteAfterRun == true { + LabeledContent("Auto-delete") { Text("after success") } + } + if let desc = job.description, !desc.isEmpty { + LabeledContent("Description") { Text(desc).font(.callout) } + } + if let agentId = job.agentId, !agentId.isEmpty { + LabeledContent("Agent") { Text(agentId) } + } + LabeledContent("Session") { Text(job.sessionTarget.rawValue) } + LabeledContent("Wake") { Text(job.wakeMode.rawValue) } + LabeledContent("Next run") { + if let date = job.nextRunDate { + Text(date.formatted(date: .abbreviated, time: .standard)) + } else { + Text("—").foregroundStyle(.secondary) + } + } + LabeledContent("Last run") { + if let date = job.lastRunDate { + Text("\(date.formatted(date: .abbreviated, time: .standard)) · \(relativeAge(from: date))") + } else { + Text("—").foregroundStyle(.secondary) + } + } + if let status = job.state.lastStatus { + LabeledContent("Last status") { Text(status) } + } + if let err = job.state.lastError, !err.isEmpty { + Text(err) + .font(.footnote) + .foregroundStyle(.orange) + .textSelection(.enabled) + } + self.payloadSummary(job.payload) + } + .frame(maxWidth: .infinity, alignment: .leading) + .padding(10) + .background(Color.secondary.opacity(0.06)) + .cornerRadius(8) + } + + func runHistoryCard(_ job: CronJob) -> some View { + VStack(alignment: .leading, spacing: 8) { + HStack { + Text("Run history") + .font(.headline) + Spacer() + Button { + Task { await self.store.refreshRuns(jobId: job.id) } + } label: { + Label("Refresh", systemImage: "arrow.clockwise") + } + .buttonStyle(.bordered) + .disabled(self.store.isLoadingRuns) + } + + if self.store.isLoadingRuns { + ProgressView().controlSize(.small) + } + + if self.store.runEntries.isEmpty { + Text("No run log entries yet.") + .font(.footnote) + .foregroundStyle(.secondary) + } else { + VStack(alignment: .leading, spacing: 6) { + ForEach(self.store.runEntries) { entry in + self.runRow(entry) + } + } + } + } + .frame(maxWidth: .infinity, alignment: .leading) + .padding(10) + .background(Color.secondary.opacity(0.06)) + .cornerRadius(8) + } + + func runRow(_ entry: CronRunLogEntry) -> some View { + VStack(alignment: .leading, spacing: 4) { + HStack(spacing: 8) { + StatusPill(text: entry.status ?? "unknown", tint: self.statusTint(entry.status)) + Text(entry.date.formatted(date: .abbreviated, time: .standard)) + .font(.caption) + .foregroundStyle(.secondary) + Spacer() + if let ms = entry.durationMs { + Text("\(ms)ms") + .font(.caption2.monospacedDigit()) + .foregroundStyle(.secondary) + } + } + if let summary = entry.summary, !summary.isEmpty { + Text(summary) + .font(.caption) + .foregroundStyle(.secondary) + .textSelection(.enabled) + .lineLimit(2) + } + if let error = entry.error, !error.isEmpty { + Text(error) + .font(.caption) + .foregroundStyle(.orange) + .textSelection(.enabled) + .lineLimit(2) + } + } + .padding(.vertical, 4) + } + + func payloadSummary(_ payload: CronPayload) -> some View { + VStack(alignment: .leading, spacing: 6) { + Text("Payload") + .font(.caption.weight(.semibold)) + .foregroundStyle(.secondary) + switch payload { + case let .systemEvent(text): + Text(text) + .font(.callout) + .textSelection(.enabled) + case let .agentTurn(message, thinking, timeoutSeconds, deliver, provider, to, _): + VStack(alignment: .leading, spacing: 4) { + Text(message) + .font(.callout) + .textSelection(.enabled) + HStack(spacing: 8) { + if let thinking, !thinking.isEmpty { StatusPill(text: "think \(thinking)", tint: .secondary) } + if let timeoutSeconds { StatusPill(text: "\(timeoutSeconds)s", tint: .secondary) } + if deliver ?? false { + StatusPill(text: "deliver", tint: .secondary) + if let provider, !provider.isEmpty { StatusPill(text: provider, tint: .secondary) } + if let to, !to.isEmpty { StatusPill(text: to, tint: .secondary) } + } + } + } + } + } + } +} diff --git a/apps/macos/Sources/OpenClaw/CronSettings+Testing.swift b/apps/macos/Sources/OpenClaw/CronSettings+Testing.swift new file mode 100644 index 0000000000000000000000000000000000000000..ffa31eb13fcdd6a03dee45a20e7470319a52c79d --- /dev/null +++ b/apps/macos/Sources/OpenClaw/CronSettings+Testing.swift @@ -0,0 +1,121 @@ +import SwiftUI + +#if DEBUG +struct CronSettings_Previews: PreviewProvider { + static var previews: some View { + let store = CronJobsStore(isPreview: true) + store.jobs = [ + CronJob( + id: "job-1", + agentId: "ops", + name: "Daily summary", + description: nil, + enabled: true, + deleteAfterRun: nil, + createdAtMs: 0, + updatedAtMs: 0, + schedule: .every(everyMs: 86_400_000, anchorMs: nil), + sessionTarget: .isolated, + wakeMode: .now, + payload: .agentTurn( + message: "Summarize inbox", + thinking: "low", + timeoutSeconds: 600, + deliver: true, + channel: "last", + to: nil, + bestEffortDeliver: true), + isolation: CronIsolation(postToMainPrefix: "Cron"), + state: CronJobState( + nextRunAtMs: Int(Date().addingTimeInterval(3600).timeIntervalSince1970 * 1000), + runningAtMs: nil, + lastRunAtMs: nil, + lastStatus: nil, + lastError: nil, + lastDurationMs: nil)), + ] + store.selectedJobId = "job-1" + store.runEntries = [ + CronRunLogEntry( + ts: Int(Date().timeIntervalSince1970 * 1000), + jobId: "job-1", + action: "finished", + status: "ok", + error: nil, + summary: "All good.", + runAtMs: nil, + durationMs: 1234, + nextRunAtMs: nil), + ] + return CronSettings(store: store, channelsStore: ChannelsStore(isPreview: true)) + .frame(width: SettingsTab.windowWidth, height: SettingsTab.windowHeight) + } +} + +@MainActor +extension CronSettings { + static func exerciseForTesting() { + let store = CronJobsStore(isPreview: true) + store.schedulerEnabled = false + store.schedulerStorePath = "/tmp/openclaw-cron-store.json" + + let job = CronJob( + id: "job-1", + agentId: "ops", + name: "Daily summary", + description: "Summary job", + enabled: true, + deleteAfterRun: nil, + createdAtMs: 1_700_000_000_000, + updatedAtMs: 1_700_000_100_000, + schedule: .cron(expr: "0 8 * * *", tz: "UTC"), + sessionTarget: .isolated, + wakeMode: .nextHeartbeat, + payload: .agentTurn( + message: "Summarize", + thinking: "low", + timeoutSeconds: 120, + deliver: true, + channel: "whatsapp", + to: "+15551234567", + bestEffortDeliver: true), + isolation: CronIsolation(postToMainPrefix: "[cron] "), + state: CronJobState( + nextRunAtMs: 1_700_000_200_000, + runningAtMs: nil, + lastRunAtMs: 1_700_000_050_000, + lastStatus: "ok", + lastError: nil, + lastDurationMs: 1200)) + + let run = CronRunLogEntry( + ts: 1_700_000_050_000, + jobId: job.id, + action: "finished", + status: "ok", + error: nil, + summary: "done", + runAtMs: 1_700_000_050_000, + durationMs: 1200, + nextRunAtMs: 1_700_000_200_000) + + store.jobs = [job] + store.selectedJobId = job.id + store.runEntries = [run] + + let view = CronSettings(store: store, channelsStore: ChannelsStore(isPreview: true)) + _ = view.body + _ = view.jobRow(job) + _ = view.jobContextMenu(job) + _ = view.detailHeader(job) + _ = view.detailCard(job) + _ = view.runHistoryCard(job) + _ = view.runRow(run) + _ = view.payloadSummary(job.payload) + _ = view.scheduleSummary(job.schedule) + _ = view.statusTint(job.state.lastStatus) + _ = view.nextRunLabel(Date()) + _ = view.formatDuration(ms: 1234) + } +} +#endif diff --git a/apps/macos/Sources/OpenClaw/CronSettings.swift b/apps/macos/Sources/OpenClaw/CronSettings.swift new file mode 100644 index 0000000000000000000000000000000000000000..999712a595d1c6ee7027a051b560a997346c3cf1 --- /dev/null +++ b/apps/macos/Sources/OpenClaw/CronSettings.swift @@ -0,0 +1,17 @@ +import Observation +import SwiftUI + +struct CronSettings: View { + @Bindable var store: CronJobsStore + @Bindable var channelsStore: ChannelsStore + @State var showEditor = false + @State var editingJob: CronJob? + @State var editorError: String? + @State var isSaving = false + @State var confirmDelete: CronJob? + + init(store: CronJobsStore = .shared, channelsStore: ChannelsStore = .shared) { + self.store = store + self.channelsStore = channelsStore + } +} diff --git a/apps/macos/Sources/OpenClaw/DebugActions.swift b/apps/macos/Sources/OpenClaw/DebugActions.swift new file mode 100644 index 0000000000000000000000000000000000000000..706d9cc2ca26a717cf0458eba09725ff6ee47dd8 --- /dev/null +++ b/apps/macos/Sources/OpenClaw/DebugActions.swift @@ -0,0 +1,265 @@ +import AppKit +import Foundation +import SwiftUI + +enum DebugActions { + private static let verboseDefaultsKey = "openclaw.debug.verboseMain" + private static let sessionMenuLimit = 12 + private static let onboardingSeenKey = "openclaw.onboardingSeen" + + @MainActor + static func openAgentEventsWindow() { + let window = NSWindow( + contentRect: NSRect(x: 0, y: 0, width: 620, height: 420), + styleMask: [.titled, .closable, .miniaturizable, .resizable], + backing: .buffered, + defer: false) + window.title = "Agent Events" + window.isReleasedWhenClosed = false + window.contentView = NSHostingView(rootView: AgentEventsWindow()) + window.center() + window.makeKeyAndOrderFront(nil) + NSApp.activate(ignoringOtherApps: true) + } + + @MainActor + static func openLog() { + let path = self.pinoLogPath() + let url = URL(fileURLWithPath: path) + guard FileManager().fileExists(atPath: path) else { + let alert = NSAlert() + alert.messageText = "Log file not found" + alert.informativeText = path + alert.runModal() + return + } + NSWorkspace.shared.activateFileViewerSelecting([url]) + } + + @MainActor + static func openConfigFolder() { + let url = OpenClawPaths.stateDirURL + NSWorkspace.shared.activateFileViewerSelecting([url]) + } + + @MainActor + static func openSessionStore() { + if AppStateStore.shared.connectionMode == .remote { + let alert = NSAlert() + alert.messageText = "Remote mode" + alert.informativeText = "Session store lives on the gateway host in remote mode." + alert.runModal() + return + } + let path = self.resolveSessionStorePath() + let url = URL(fileURLWithPath: path) + if FileManager().fileExists(atPath: path) { + NSWorkspace.shared.activateFileViewerSelecting([url]) + } else { + NSWorkspace.shared.open(url.deletingLastPathComponent()) + } + } + + static func sendTestNotification() async { + _ = await NotificationManager().send(title: "OpenClaw", body: "Test notification", sound: nil) + } + + static func sendDebugVoice() async -> Result { + let message = """ + This is a debug test from the Mac app. Reply with "Debug test works (and a funny pun)" \ + if you received that. + """ + let result = await VoiceWakeForwarder.forward(transcript: message) + switch result { + case .success: + return .success("Sent. Await reply.") + case let .failure(error): + let detail = error.localizedDescription.trimmingCharacters(in: .whitespacesAndNewlines) + return .failure(.message("Send failed: \(detail)")) + } + } + + static func restartGateway() { + Task { @MainActor in + switch AppStateStore.shared.connectionMode { + case .local: + GatewayProcessManager.shared.stop() + // Kick the control channel + health check so the UI recovers immediately. + await GatewayConnection.shared.shutdown() + try? await Task.sleep(nanoseconds: 300_000_000) + GatewayProcessManager.shared.setActive(true) + Task { try? await ControlChannel.shared.configure(mode: .local) } + Task { await HealthStore.shared.refresh(onDemand: true) } + + case .remote: + // In remote mode, there is no local gateway to restart. "Restart Gateway" should + // reset the SSH control tunnel + reconnect so the menu recovers. + await RemoteTunnelManager.shared.stopAll() + await GatewayConnection.shared.shutdown() + do { + _ = try await RemoteTunnelManager.shared.ensureControlTunnel() + let settings = CommandResolver.connectionSettings() + try await ControlChannel.shared.configure(mode: .remote( + target: settings.target, + identity: settings.identity)) + } catch { + // ControlChannel will surface a degraded state; also refresh health to update the menu text. + Task { await HealthStore.shared.refresh(onDemand: true) } + } + + case .unconfigured: + await GatewayConnection.shared.shutdown() + await ControlChannel.shared.disconnect() + } + } + } + + static func resetGatewayTunnel() async -> Result { + let mode = CommandResolver.connectionSettings().mode + guard mode == .remote else { + return .failure(.message("Remote mode is not enabled.")) + } + await RemoteTunnelManager.shared.stopAll() + await GatewayConnection.shared.shutdown() + do { + _ = try await RemoteTunnelManager.shared.ensureControlTunnel() + let settings = CommandResolver.connectionSettings() + try await ControlChannel.shared.configure(mode: .remote( + target: settings.target, + identity: settings.identity)) + await HealthStore.shared.refresh(onDemand: true) + return .success("SSH tunnel reset.") + } catch { + Task { await HealthStore.shared.refresh(onDemand: true) } + return .failure(.message(error.localizedDescription)) + } + } + + static func pinoLogPath() -> String { + LogLocator.bestLogFile()?.path ?? LogLocator.launchdLogPath + } + + @MainActor + static func runHealthCheckNow() async { + await HealthStore.shared.refresh(onDemand: true) + } + + static func sendTestHeartbeat() async -> Result { + do { + _ = await GatewayConnection.shared.setHeartbeatsEnabled(true) + await ControlChannel.shared.configure() + let data = try await ControlChannel.shared.request(method: "last-heartbeat") + if let evt = try? JSONDecoder().decode(ControlHeartbeatEvent.self, from: data) { + return .success(evt) + } + return .success(nil) + } catch { + return .failure(error) + } + } + + static var verboseLoggingEnabledMain: Bool { + UserDefaults.standard.bool(forKey: self.verboseDefaultsKey) + } + + static func toggleVerboseLoggingMain() async -> Bool { + let newValue = !self.verboseLoggingEnabledMain + UserDefaults.standard.set(newValue, forKey: self.verboseDefaultsKey) + _ = try? await ControlChannel.shared.request( + method: "system-event", + params: ["text": AnyHashable("verbose-main:\(newValue ? "on" : "off")")]) + return newValue + } + + @MainActor + static func restartApp() { + let url = Bundle.main.bundleURL + let task = Process() + // Relaunch shortly after this instance exits so we get a true restart even in debug. + task.launchPath = "/bin/sh" + task.arguments = ["-c", "sleep 0.2; open -n \"$1\"", "_", url.path] + try? task.run() + NSApp.terminate(nil) + } + + @MainActor + static func restartOnboarding() { + UserDefaults.standard.set(false, forKey: self.onboardingSeenKey) + UserDefaults.standard.set(0, forKey: onboardingVersionKey) + AppStateStore.shared.onboardingSeen = false + OnboardingController.shared.restart() + } + + @MainActor + private static func resolveSessionStorePath() -> String { + let defaultPath = SessionLoader.defaultStorePath + let configURL = OpenClawPaths.configURL + guard + let data = try? Data(contentsOf: configURL), + let parsed = try? JSONSerialization.jsonObject(with: data) as? [String: Any], + let session = parsed["session"] as? [String: Any], + let path = session["store"] as? String, + !path.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty + else { + return defaultPath + } + return path + } + + // MARK: - Sessions (thinking / verbose) + + static func recentSessions(limit: Int = sessionMenuLimit) async -> [SessionRow] { + guard let snapshot = try? await SessionLoader.loadSnapshot(limit: limit) else { return [] } + return Array(snapshot.rows.prefix(limit)) + } + + static func updateSession( + key: String, + thinking: String?, + verbose: String?) async throws + { + var params: [String: AnyHashable] = ["key": AnyHashable(key)] + params["thinkingLevel"] = thinking.map(AnyHashable.init) ?? AnyHashable(NSNull()) + params["verboseLevel"] = verbose.map(AnyHashable.init) ?? AnyHashable(NSNull()) + _ = try await ControlChannel.shared.request(method: "sessions.patch", params: params) + } + + // MARK: - Port diagnostics + + typealias PortListener = PortGuardian.ReportListener + typealias PortReport = PortGuardian.PortReport + + static func checkGatewayPorts() async -> [PortReport] { + let mode = CommandResolver.connectionSettings().mode + return await PortGuardian.shared.diagnose(mode: mode) + } + + static func killProcess(_ pid: Int) async -> Result { + let primary = await ShellExecutor.run(command: ["kill", "-TERM", "\(pid)"], cwd: nil, env: nil, timeout: 2) + if primary.ok { return .success(()) } + let force = await ShellExecutor.run(command: ["kill", "-KILL", "\(pid)"], cwd: nil, env: nil, timeout: 2) + if force.ok { return .success(()) } + let detail = force.message ?? primary.message ?? "kill failed" + return .failure(.message(detail)) + } + + @MainActor + static func openSessionStoreInCode() { + let path = SessionLoader.defaultStorePath + let proc = Process() + proc.launchPath = "/usr/bin/env" + proc.arguments = ["code", path] + try? proc.run() + } +} + +enum DebugActionError: LocalizedError { + case message(String) + + var errorDescription: String? { + switch self { + case let .message(text): + text + } + } +} diff --git a/apps/macos/Sources/OpenClaw/DebugSettings.swift b/apps/macos/Sources/OpenClaw/DebugSettings.swift new file mode 100644 index 0000000000000000000000000000000000000000..678ffc9e3ff5e599c564d0a6e08b32640e3d24ce --- /dev/null +++ b/apps/macos/Sources/OpenClaw/DebugSettings.swift @@ -0,0 +1,1026 @@ +import AppKit +import Observation +import SwiftUI +import UniformTypeIdentifiers + +struct DebugSettings: View { + @Bindable var state: AppState + private let isPreview = ProcessInfo.processInfo.isPreview + private let labelColumnWidth: CGFloat = 140 + @AppStorage(modelCatalogPathKey) private var modelCatalogPath: String = ModelCatalogLoader.defaultPath + @AppStorage(modelCatalogReloadKey) private var modelCatalogReloadBump: Int = 0 + @AppStorage(iconOverrideKey) private var iconOverrideRaw: String = IconOverrideSelection.system.rawValue + @AppStorage(canvasEnabledKey) private var canvasEnabled: Bool = true + @State private var modelsCount: Int? + @State private var modelsLoading = false + @State private var modelsError: String? + private let gatewayManager = GatewayProcessManager.shared + private let healthStore = HealthStore.shared + @State private var launchAgentWriteDisabled = GatewayLaunchAgentManager.isLaunchAgentWriteDisabled() + @State private var launchAgentWriteError: String? + @State private var gatewayRootInput: String = GatewayProcessManager.shared.projectRootPath() + @State private var sessionStorePath: String = SessionLoader.defaultStorePath + @State private var sessionStoreSaveError: String? + @State private var debugSendInFlight = false + @State private var debugSendStatus: String? + @State private var debugSendError: String? + @State private var portCheckInFlight = false + @State private var portReports: [DebugActions.PortReport] = [] + @State private var portKillStatus: String? + @State private var tunnelResetInFlight = false + @State private var tunnelResetStatus: String? + @State private var pendingKill: DebugActions.PortListener? + @AppStorage(debugFileLogEnabledKey) private var diagnosticsFileLogEnabled: Bool = false + @AppStorage(appLogLevelKey) private var appLogLevelRaw: String = AppLogLevel.default.rawValue + + @State private var canvasSessionKey: String = "main" + @State private var canvasStatus: String? + @State private var canvasError: String? + @State private var canvasEvalJS: String = "document.title" + @State private var canvasEvalResult: String? + @State private var canvasSnapshotPath: String? + + init(state: AppState = AppStateStore.shared) { + self.state = state + } + + var body: some View { + ScrollView(.vertical) { + VStack(alignment: .leading, spacing: 14) { + self.header + + self.launchdSection + self.appInfoSection + self.gatewaySection + self.logsSection + self.portsSection + self.pathsSection + self.quickActionsSection + self.canvasSection + self.experimentsSection + + Spacer(minLength: 0) + } + .frame(maxWidth: .infinity, alignment: .leading) + .padding(.horizontal, 24) + .padding(.vertical, 18) + .groupBoxStyle(PlainSettingsGroupBoxStyle()) + } + .task { + guard !self.isPreview else { return } + await self.reloadModels() + self.loadSessionStorePath() + } + .alert(item: self.$pendingKill) { listener in + Alert( + title: Text("Kill \(listener.command) (\(listener.pid))?"), + message: Text("This process looks expected for the current mode. Kill anyway?"), + primaryButton: .destructive(Text("Kill")) { + Task { await self.killConfirmed(listener.pid) } + }, + secondaryButton: .cancel()) + } + } + + private var launchdSection: some View { + GroupBox("Gateway startup") { + VStack(alignment: .leading, spacing: 8) { + Toggle("Attach only (skip launchd install)", isOn: self.$launchAgentWriteDisabled) + .onChange(of: self.launchAgentWriteDisabled) { _, newValue in + self.launchAgentWriteError = GatewayLaunchAgentManager.setLaunchAgentWriteDisabled(newValue) + if self.launchAgentWriteError != nil { + self.launchAgentWriteDisabled = GatewayLaunchAgentManager.isLaunchAgentWriteDisabled() + return + } + if newValue { + Task { + _ = await GatewayLaunchAgentManager.set( + enabled: false, + bundlePath: Bundle.main.bundlePath, + port: GatewayEnvironment.gatewayPort()) + } + } + } + + Text( + "When enabled, OpenClaw won't install or manage \(gatewayLaunchdLabel). " + + "It will only attach to an existing Gateway.") + .font(.caption) + .foregroundStyle(.secondary) + + if let launchAgentWriteError { + Text(launchAgentWriteError) + .font(.caption) + .foregroundStyle(.red) + } + } + } + } + + private var header: some View { + VStack(alignment: .leading, spacing: 6) { + Text("Debug") + .font(.title3.weight(.semibold)) + Text("Tools for diagnosing local issues (Gateway, ports, logs, Canvas).") + .font(.callout) + .foregroundStyle(.secondary) + } + } + + private func gridLabel(_ text: String) -> some View { + Text(text) + .foregroundStyle(.secondary) + .frame(width: self.labelColumnWidth, alignment: .leading) + } + + private var appInfoSection: some View { + GroupBox("App") { + Grid(alignment: .leadingFirstTextBaseline, horizontalSpacing: 14, verticalSpacing: 10) { + GridRow { + self.gridLabel("Health") + HStack(spacing: 8) { + Circle().fill(self.healthStore.state.tint).frame(width: 10, height: 10) + Text(self.healthStore.summaryLine) + } + .frame(maxWidth: .infinity, alignment: .leading) + } + GridRow { + self.gridLabel("CLI") + let loc = CLIInstaller.installedLocation() + Text(loc ?? "missing") + .font(.caption.monospaced()) + .foregroundStyle(loc == nil ? Color.red : Color.secondary) + .textSelection(.enabled) + .lineLimit(1) + .truncationMode(.middle) + } + GridRow { + self.gridLabel("PID") + Text("\(ProcessInfo.processInfo.processIdentifier)") + } + GridRow { + self.gridLabel("Binary path") + Text(Bundle.main.bundlePath) + .font(.caption2.monospaced()) + .foregroundStyle(.secondary) + .textSelection(.enabled) + .lineLimit(1) + .truncationMode(.middle) + } + } + } + } + + private var gatewaySection: some View { + GroupBox("Gateway") { + VStack(alignment: .leading, spacing: 10) { + Grid(alignment: .leadingFirstTextBaseline, horizontalSpacing: 14, verticalSpacing: 10) { + GridRow { + self.gridLabel("Status") + HStack(spacing: 8) { + Text(self.gatewayManager.status.label) + } + .frame(maxWidth: .infinity, alignment: .leading) + } + } + + let key = DeepLinkHandler.currentKey() + HStack(spacing: 8) { + Text("Key") + .foregroundStyle(.secondary) + .frame(width: self.labelColumnWidth, alignment: .leading) + Text(key) + .font(.caption2.monospaced()) + .foregroundStyle(.secondary) + .textSelection(.enabled) + .lineLimit(1) + .truncationMode(.middle) + Button("Copy") { + NSPasteboard.general.clearContents() + NSPasteboard.general.setString(key, forType: .string) + } + .buttonStyle(.bordered) + Button("Copy sample URL") { + let msg = "Hello from deep link" + let encoded = msg.addingPercentEncoding(withAllowedCharacters: .urlQueryAllowed) ?? msg + let url = "openclaw://agent?message=\(encoded)&key=\(key)" + NSPasteboard.general.clearContents() + NSPasteboard.general.setString(url, forType: .string) + } + .buttonStyle(.bordered) + Spacer(minLength: 0) + } + + Text("Deep links (openclaw://…) are always enabled; the key controls unattended runs.") + .font(.caption2) + .foregroundStyle(.secondary) + + VStack(alignment: .leading, spacing: 6) { + Text("Stdout / stderr") + .font(.caption.weight(.semibold)) + ScrollView { + Text(self.gatewayManager.log.isEmpty ? "—" : self.gatewayManager.log) + .font(.caption.monospaced()) + .frame(maxWidth: .infinity, alignment: .leading) + .textSelection(.enabled) + } + .frame(height: 180) + .overlay(RoundedRectangle(cornerRadius: 6).stroke(Color.secondary.opacity(0.2))) + + HStack(spacing: 8) { + if self.canRestartGateway { + Button("Restart Gateway") { DebugActions.restartGateway() } + } + Button("Clear log") { GatewayProcessManager.shared.clearLog() } + Spacer(minLength: 0) + } + .buttonStyle(.bordered) + } + } + } + } + + private var logsSection: some View { + GroupBox("Logs") { + Grid(alignment: .leadingFirstTextBaseline, horizontalSpacing: 14, verticalSpacing: 10) { + GridRow { + self.gridLabel("Pino log") + VStack(alignment: .leading, spacing: 6) { + HStack(spacing: 8) { + Button("Open") { DebugActions.openLog() } + .buttonStyle(.bordered) + Text(DebugActions.pinoLogPath()) + .font(.caption2.monospaced()) + .foregroundStyle(.secondary) + .textSelection(.enabled) + .lineLimit(1) + .truncationMode(.middle) + } + } + } + + GridRow { + self.gridLabel("App logging") + VStack(alignment: .leading, spacing: 8) { + Picker("Verbosity", selection: self.$appLogLevelRaw) { + ForEach(AppLogLevel.allCases) { level in + Text(level.title).tag(level.rawValue) + } + } + .pickerStyle(.menu) + .labelsHidden() + .help("Controls the macOS app log verbosity.") + + Toggle("Write rolling diagnostics log (JSONL)", isOn: self.$diagnosticsFileLogEnabled) + .toggleStyle(.checkbox) + .help( + "Writes a rotating, local-only log under ~/Library/Logs/OpenClaw/. " + + "Enable only while actively debugging.") + + HStack(spacing: 8) { + Button("Open folder") { + NSWorkspace.shared.open(DiagnosticsFileLog.logDirectoryURL()) + } + .buttonStyle(.bordered) + Button("Clear") { + Task { try? await DiagnosticsFileLog.shared.clear() } + } + .buttonStyle(.bordered) + } + Text(DiagnosticsFileLog.logFileURL().path) + .font(.caption2.monospaced()) + .foregroundStyle(.secondary) + .textSelection(.enabled) + .lineLimit(1) + .truncationMode(.middle) + } + } + } + } + } + + private var portsSection: some View { + GroupBox("Ports") { + VStack(alignment: .leading, spacing: 10) { + HStack(spacing: 8) { + Text("Port diagnostics") + .font(.caption.weight(.semibold)) + if self.portCheckInFlight { ProgressView().controlSize(.small) } + Spacer() + Button("Check gateway ports") { + Task { await self.runPortCheck() } + } + .buttonStyle(.borderedProminent) + .disabled(self.portCheckInFlight) + Button("Reset SSH tunnel") { + Task { await self.resetGatewayTunnel() } + } + .buttonStyle(.bordered) + .disabled(self.tunnelResetInFlight || !self.isRemoteMode) + } + + if let portKillStatus { + Text(portKillStatus) + .font(.caption2) + .foregroundStyle(.secondary) + .fixedSize(horizontal: false, vertical: true) + } + if let tunnelResetStatus { + Text(tunnelResetStatus) + .font(.caption2) + .foregroundStyle(.secondary) + .fixedSize(horizontal: false, vertical: true) + } + + if self.portReports.isEmpty, !self.portCheckInFlight { + Text("Check which process owns \(GatewayEnvironment.gatewayPort()) and suggest fixes.") + .font(.caption2) + .foregroundStyle(.secondary) + } else { + ForEach(self.portReports) { report in + VStack(alignment: .leading, spacing: 4) { + Text("Port \(report.port)") + .font(.footnote.weight(.semibold)) + Text(report.summary) + .font(.caption) + .foregroundStyle(.secondary) + .fixedSize(horizontal: false, vertical: true) + ForEach(report.listeners) { listener in + VStack(alignment: .leading, spacing: 2) { + HStack(spacing: 8) { + Text("\(listener.command) (\(listener.pid))") + .font(.caption.monospaced()) + .foregroundStyle(listener.expected ? .secondary : Color.red) + .lineLimit(1) + Spacer() + Button("Kill") { + self.requestKill(listener) + } + .buttonStyle(.bordered) + } + Text(listener.fullCommand) + .font(.caption2.monospaced()) + .foregroundStyle(.secondary) + .lineLimit(2) + .truncationMode(.middle) + } + .padding(6) + .background(Color.secondary.opacity(0.05)) + .cornerRadius(4) + } + } + .padding(8) + .background(Color.secondary.opacity(0.08)) + .cornerRadius(6) + } + } + } + } + } + + private var pathsSection: some View { + GroupBox("Paths") { + VStack(alignment: .leading, spacing: 12) { + VStack(alignment: .leading, spacing: 6) { + Text("OpenClaw project root") + .font(.caption.weight(.semibold)) + HStack(spacing: 8) { + TextField("Path to openclaw repo", text: self.$gatewayRootInput) + .textFieldStyle(.roundedBorder) + .font(.caption.monospaced()) + .onSubmit { self.saveRelayRoot() } + Button("Save") { self.saveRelayRoot() } + .buttonStyle(.borderedProminent) + Button("Reset") { + let def = FileManager().homeDirectoryForCurrentUser + .appendingPathComponent("Projects/openclaw").path + self.gatewayRootInput = def + self.saveRelayRoot() + } + .buttonStyle(.bordered) + } + Text("Used for pnpm/node fallback and PATH population when launching the gateway.") + .font(.caption2) + .foregroundStyle(.secondary) + } + + Divider() + + Grid(alignment: .leadingFirstTextBaseline, horizontalSpacing: 14, verticalSpacing: 10) { + GridRow { + self.gridLabel("Session store") + VStack(alignment: .leading, spacing: 6) { + HStack(spacing: 8) { + TextField("Path", text: self.$sessionStorePath) + .textFieldStyle(.roundedBorder) + .font(.caption.monospaced()) + .frame(width: 360) + Button("Save") { self.saveSessionStorePath() } + .buttonStyle(.borderedProminent) + } + if let sessionStoreSaveError { + Text(sessionStoreSaveError) + .font(.footnote) + .foregroundStyle(.secondary) + } else { + Text("Used by the CLI session loader; stored in ~/.openclaw/openclaw.json.") + .font(.footnote) + .foregroundStyle(.secondary) + } + } + } + GridRow { + self.gridLabel("Model catalog") + VStack(alignment: .leading, spacing: 6) { + Text(self.modelCatalogPath) + .font(.caption.monospaced()) + .foregroundStyle(.secondary) + .lineLimit(2) + HStack(spacing: 8) { + Button { + self.chooseCatalogFile() + } label: { + Label("Choose models.generated.ts…", systemImage: "folder") + } + .buttonStyle(.bordered) + + Button { + Task { await self.reloadModels() } + } label: { + Label( + self.modelsLoading ? "Reloading…" : "Reload models", + systemImage: "arrow.clockwise") + } + .buttonStyle(.bordered) + .disabled(self.modelsLoading) + } + if let modelsError { + Text(modelsError) + .font(.footnote) + .foregroundStyle(.secondary) + } else if let modelsCount { + Text("Loaded \(modelsCount) models") + .font(.footnote) + .foregroundStyle(.secondary) + } + Text("Local fallback for model picker when gateway models.list is unavailable.") + .font(.footnote) + .foregroundStyle(.tertiary) + } + } + } + } + } + } + + private var quickActionsSection: some View { + GroupBox("Quick actions") { + VStack(alignment: .leading, spacing: 10) { + HStack(spacing: 8) { + Button("Send Test Notification") { + Task { await DebugActions.sendTestNotification() } + } + .buttonStyle(.bordered) + + Button("Open Agent Events") { + DebugActions.openAgentEventsWindow() + } + .buttonStyle(.borderedProminent) + + Spacer(minLength: 0) + } + + VStack(alignment: .leading, spacing: 6) { + Button { + Task { await self.sendVoiceDebug() } + } label: { + Label( + self.debugSendInFlight ? "Sending debug voice…" : "Send debug voice", + systemImage: self.debugSendInFlight ? "bolt.horizontal.circle" : "waveform") + } + .buttonStyle(.borderedProminent) + .disabled(self.debugSendInFlight) + + if !self.debugSendInFlight { + if let debugSendStatus { + Text(debugSendStatus) + .font(.caption) + .foregroundStyle(.secondary) + } else if let debugSendError { + Text(debugSendError) + .font(.caption) + .foregroundStyle(.red) + } else { + Text( + """ + Uses the Voice Wake path: forwards over SSH when configured, + otherwise runs locally via rpc. + """) + .font(.caption) + .foregroundStyle(.secondary) + } + } + } + + VStack(alignment: .leading, spacing: 6) { + Text( + "Note: macOS may require restarting OpenClaw after enabling Accessibility or Screen Recording.") + .font(.caption) + .foregroundStyle(.secondary) + .fixedSize(horizontal: false, vertical: true) + + Button { + LaunchdManager.startOpenClaw() + } label: { + Label("Restart OpenClaw", systemImage: "arrow.counterclockwise") + } + .buttonStyle(.bordered) + .controlSize(.small) + } + + HStack(spacing: 8) { + Button("Restart app") { DebugActions.restartApp() } + Button("Restart onboarding") { DebugActions.restartOnboarding() } + Button("Reveal app in Finder") { self.revealApp() } + Spacer(minLength: 0) + } + .buttonStyle(.bordered) + } + } + } + + private var canvasSection: some View { + GroupBox("Canvas") { + VStack(alignment: .leading, spacing: 10) { + Text("Enable/disable Canvas in General settings.") + .font(.caption) + .foregroundStyle(.secondary) + + HStack(spacing: 8) { + TextField("Session", text: self.$canvasSessionKey) + .textFieldStyle(.roundedBorder) + .font(.caption.monospaced()) + .frame(width: 160) + Button("Show panel") { + Task { await self.canvasPresent() } + } + .buttonStyle(.borderedProminent) + Button("Hide panel") { + CanvasManager.shared.hideAll() + self.canvasStatus = "hidden" + self.canvasError = nil + } + .buttonStyle(.bordered) + Button("Write sample page") { + Task { await self.canvasWriteSamplePage() } + } + .buttonStyle(.bordered) + Spacer(minLength: 0) + } + + HStack(spacing: 8) { + TextField("Eval JS", text: self.$canvasEvalJS) + .textFieldStyle(.roundedBorder) + .font(.caption.monospaced()) + .frame(maxWidth: 520) + Button("Eval") { + Task { await self.canvasEval() } + } + .buttonStyle(.bordered) + Button("Snapshot") { + Task { await self.canvasSnapshot() } + } + .buttonStyle(.bordered) + Spacer(minLength: 0) + } + + if let canvasStatus { + Text(canvasStatus) + .font(.caption2.monospaced()) + .foregroundStyle(.secondary) + .textSelection(.enabled) + } + if let canvasEvalResult { + Text("eval → \(canvasEvalResult)") + .font(.caption2.monospaced()) + .foregroundStyle(.secondary) + .lineLimit(2) + .truncationMode(.middle) + .textSelection(.enabled) + } + if let canvasSnapshotPath { + HStack(spacing: 8) { + Text("snapshot → \(canvasSnapshotPath)") + .font(.caption2.monospaced()) + .foregroundStyle(.secondary) + .lineLimit(1) + .truncationMode(.middle) + .textSelection(.enabled) + Button("Reveal") { + NSWorkspace.shared + .activateFileViewerSelecting([URL(fileURLWithPath: canvasSnapshotPath)]) + } + .buttonStyle(.bordered) + Spacer(minLength: 0) + } + } + if let canvasError { + Text(canvasError) + .font(.caption2) + .foregroundStyle(.red) + } else { + Text("Tip: the session directory is returned by “Show panel”.") + .font(.caption2) + .foregroundStyle(.tertiary) + } + } + } + } + + private var experimentsSection: some View { + GroupBox("Experiments") { + Grid(alignment: .leadingFirstTextBaseline, horizontalSpacing: 14, verticalSpacing: 10) { + GridRow { + self.gridLabel("Icon override") + Picker("", selection: self.bindingOverride) { + ForEach(IconOverrideSelection.allCases) { option in + Text(option.label).tag(option.rawValue) + } + } + .labelsHidden() + .frame(maxWidth: 280, alignment: .leading) + } + GridRow { + self.gridLabel("Chat") + Text("Native SwiftUI") + .font(.callout) + .foregroundStyle(.secondary) + } + } + } + } + + @MainActor + private func runPortCheck() async { + self.portCheckInFlight = true + self.portKillStatus = nil + let reports = await DebugActions.checkGatewayPorts() + self.portReports = reports + self.portCheckInFlight = false + } + + @MainActor + private func resetGatewayTunnel() async { + self.tunnelResetInFlight = true + self.tunnelResetStatus = nil + let result = await DebugActions.resetGatewayTunnel() + switch result { + case let .success(message): + self.tunnelResetStatus = message + case let .failure(err): + self.tunnelResetStatus = err.localizedDescription + } + await self.runPortCheck() + self.tunnelResetInFlight = false + } + + @MainActor + private func requestKill(_ listener: DebugActions.PortListener) { + if listener.expected { + self.pendingKill = listener + } else { + Task { await self.killConfirmed(listener.pid) } + } + } + + @MainActor + private func killConfirmed(_ pid: Int32) async { + let result = await DebugActions.killProcess(Int(pid)) + switch result { + case .success: + self.portKillStatus = "Sent kill to \(pid)." + await self.runPortCheck() + case let .failure(err): + self.portKillStatus = "Kill \(pid) failed: \(err.localizedDescription)" + } + } + + private func chooseCatalogFile() { + let panel = NSOpenPanel() + panel.title = "Select models.generated.ts" + let tsType = UTType(filenameExtension: "ts") + ?? UTType(tag: "ts", tagClass: .filenameExtension, conformingTo: .sourceCode) + ?? .item + panel.allowedContentTypes = [tsType] + panel.allowsMultipleSelection = false + panel.directoryURL = URL(fileURLWithPath: self.modelCatalogPath).deletingLastPathComponent() + if panel.runModal() == .OK, let url = panel.url { + self.modelCatalogPath = url.path + self.modelCatalogReloadBump += 1 + Task { await self.reloadModels() } + } + } + + private func reloadModels() async { + guard !self.modelsLoading else { return } + self.modelsLoading = true + self.modelsError = nil + self.modelCatalogReloadBump += 1 + defer { self.modelsLoading = false } + do { + let loaded = try await ModelCatalogLoader.load(from: self.modelCatalogPath) + self.modelsCount = loaded.count + } catch { + self.modelsCount = nil + self.modelsError = error.localizedDescription + } + } + + private func sendVoiceDebug() async { + await MainActor.run { + self.debugSendInFlight = true + self.debugSendError = nil + self.debugSendStatus = nil + } + + let result = await DebugActions.sendDebugVoice() + + await MainActor.run { + self.debugSendInFlight = false + switch result { + case let .success(message): + self.debugSendStatus = message + self.debugSendError = nil + case let .failure(error): + self.debugSendStatus = nil + self.debugSendError = error.localizedDescription + } + } + } + + private func revealApp() { + let url = Bundle.main.bundleURL + NSWorkspace.shared.activateFileViewerSelecting([url]) + } + + private func saveRelayRoot() { + GatewayProcessManager.shared.setProjectRoot(path: self.gatewayRootInput) + } + + private func loadSessionStorePath() { + let url = self.configURL() + guard + let data = try? Data(contentsOf: url), + let parsed = try? JSONSerialization.jsonObject(with: data) as? [String: Any], + let session = parsed["session"] as? [String: Any], + let path = session["store"] as? String + else { + self.sessionStorePath = SessionLoader.defaultStorePath + return + } + self.sessionStorePath = path + } + + private func saveSessionStorePath() { + let trimmed = self.sessionStorePath.trimmingCharacters(in: .whitespacesAndNewlines) + var root: [String: Any] = [:] + let url = self.configURL() + if let data = try? Data(contentsOf: url), + let parsed = try? JSONSerialization.jsonObject(with: data) as? [String: Any] + { + root = parsed + } + + var session = root["session"] as? [String: Any] ?? [:] + session["store"] = trimmed.isEmpty ? SessionLoader.defaultStorePath : trimmed + root["session"] = session + + do { + let data = try JSONSerialization.data(withJSONObject: root, options: [.prettyPrinted, .sortedKeys]) + try FileManager().createDirectory( + at: url.deletingLastPathComponent(), + withIntermediateDirectories: true) + try data.write(to: url, options: [.atomic]) + self.sessionStoreSaveError = nil + } catch { + self.sessionStoreSaveError = error.localizedDescription + } + } + + private var bindingOverride: Binding { + Binding { + self.iconOverrideRaw + } set: { newValue in + self.iconOverrideRaw = newValue + if let selection = IconOverrideSelection(rawValue: newValue) { + Task { @MainActor in + AppStateStore.shared.iconOverride = selection + WorkActivityStore.shared.resolveIconState(override: selection) + } + } + } + } + + private var isRemoteMode: Bool { + CommandResolver.connectionSettings().mode == .remote + } + + private var canRestartGateway: Bool { + self.state.connectionMode == .local + } + + private func configURL() -> URL { + OpenClawPaths.configURL + } +} + +extension DebugSettings { + // MARK: - Canvas debug actions + + @MainActor + private func canvasPresent() async { + self.canvasError = nil + let session = self.canvasSessionKey.trimmingCharacters(in: .whitespacesAndNewlines) + do { + let dir = try CanvasManager.shared.show(sessionKey: session.isEmpty ? "main" : session, path: "/") + self.canvasStatus = "dir: \(dir)" + } catch { + self.canvasError = error.localizedDescription + } + } + + @MainActor + private func canvasWriteSamplePage() async { + self.canvasError = nil + let session = self.canvasSessionKey.trimmingCharacters(in: .whitespacesAndNewlines) + do { + let dir = try CanvasManager.shared.show(sessionKey: session.isEmpty ? "main" : session, path: "/") + let url = URL(fileURLWithPath: dir).appendingPathComponent("index.html", isDirectory: false) + let now = ISO8601DateFormatter().string(from: Date()) + let html = """ + + + + + + Canvas Debug + + + +
+
+
Canvas Debug
+
generated: \(now)
+
userAgent:
+ +
count: 0
+
+
+
This is a local file served by the WKURLSchemeHandler.
+
+
+
+
+
+
+ + + + """ + try html.write(to: url, atomically: true, encoding: .utf8) + self.canvasStatus = "wrote: \(url.path)" + _ = try CanvasManager.shared.show(sessionKey: session.isEmpty ? "main" : session, path: "/") + } catch { + self.canvasError = error.localizedDescription + } + } + + @MainActor + private func canvasEval() async { + self.canvasError = nil + self.canvasEvalResult = nil + do { + let session = self.canvasSessionKey.trimmingCharacters(in: .whitespacesAndNewlines) + let result = try await CanvasManager.shared.eval( + sessionKey: session.isEmpty ? "main" : session, + javaScript: self.canvasEvalJS) + self.canvasEvalResult = result + } catch { + self.canvasError = error.localizedDescription + } + } + + @MainActor + private func canvasSnapshot() async { + self.canvasError = nil + self.canvasSnapshotPath = nil + do { + let session = self.canvasSessionKey.trimmingCharacters(in: .whitespacesAndNewlines) + let path = try await CanvasManager.shared.snapshot( + sessionKey: session.isEmpty ? "main" : session, + outPath: nil) + self.canvasSnapshotPath = path + } catch { + self.canvasError = error.localizedDescription + } + } +} + +struct PlainSettingsGroupBoxStyle: GroupBoxStyle { + func makeBody(configuration: Configuration) -> some View { + VStack(alignment: .leading, spacing: 10) { + configuration.label + .font(.caption.weight(.semibold)) + .foregroundStyle(.secondary) + configuration.content + } + .frame(maxWidth: .infinity, alignment: .leading) + } +} + +#if DEBUG +struct DebugSettings_Previews: PreviewProvider { + static var previews: some View { + DebugSettings(state: .preview) + .frame(width: SettingsTab.windowWidth, height: SettingsTab.windowHeight) + } +} + +@MainActor +extension DebugSettings { + static func exerciseForTesting() async { + let view = DebugSettings(state: .preview) + view.modelsCount = 3 + view.modelsLoading = false + view.modelsError = "Failed to load models" + view.gatewayRootInput = "/tmp/openclaw" + view.sessionStorePath = "/tmp/sessions.json" + view.sessionStoreSaveError = "Save failed" + view.debugSendInFlight = true + view.debugSendStatus = "Sent" + view.debugSendError = "Failed" + view.portCheckInFlight = true + view.portReports = [ + DebugActions.PortReport( + port: GatewayEnvironment.gatewayPort(), + expected: "Gateway websocket (node/tsx)", + status: .missing("Missing"), + listeners: []), + ] + view.portKillStatus = "Killed" + view.pendingKill = DebugActions.PortListener( + pid: 1, + command: "node", + fullCommand: "node", + user: nil, + expected: true) + view.canvasSessionKey = "main" + view.canvasStatus = "Canvas ok" + view.canvasError = "Canvas error" + view.canvasEvalJS = "document.title" + view.canvasEvalResult = "Canvas" + view.canvasSnapshotPath = "/tmp/snapshot.png" + + _ = view.body + _ = view.header + _ = view.appInfoSection + _ = view.gatewaySection + _ = view.logsSection + _ = view.portsSection + _ = view.pathsSection + _ = view.quickActionsSection + _ = view.canvasSection + _ = view.experimentsSection + _ = view.gridLabel("Test") + + view.loadSessionStorePath() + await view.reloadModels() + } +} +#endif diff --git a/apps/macos/Sources/OpenClaw/DeepLinks.swift b/apps/macos/Sources/OpenClaw/DeepLinks.swift new file mode 100644 index 0000000000000000000000000000000000000000..13543e658b32741861476274219203c57f13d02b --- /dev/null +++ b/apps/macos/Sources/OpenClaw/DeepLinks.swift @@ -0,0 +1,151 @@ +import AppKit +import OpenClawKit +import Foundation +import OSLog +import Security + +private let deepLinkLogger = Logger(subsystem: "ai.openclaw", category: "DeepLink") + +@MainActor +final class DeepLinkHandler { + static let shared = DeepLinkHandler() + + private var lastPromptAt: Date = .distantPast + + // Ephemeral, in-memory key used for unattended deep links originating from the in-app Canvas. + // This avoids blocking Canvas init on UserDefaults and doesn't weaken the external deep-link prompt: + // outside callers can't know this randomly generated key. + private nonisolated static let canvasUnattendedKey: String = DeepLinkHandler.generateRandomKey() + + func handle(url: URL) async { + guard let route = DeepLinkParser.parse(url) else { + deepLinkLogger.debug("ignored url \(url.absoluteString, privacy: .public)") + return + } + guard !AppStateStore.shared.isPaused else { + self.presentAlert(title: "OpenClaw is paused", message: "Unpause OpenClaw to run agent actions.") + return + } + + switch route { + case let .agent(link): + await self.handleAgent(link: link, originalURL: url) + } + } + + private func handleAgent(link: AgentDeepLink, originalURL: URL) async { + let messagePreview = link.message.trimmingCharacters(in: .whitespacesAndNewlines) + if messagePreview.count > 20000 { + self.presentAlert(title: "Deep link too large", message: "Message exceeds 20,000 characters.") + return + } + + let allowUnattended = link.key == Self.canvasUnattendedKey || link.key == Self.expectedKey() + if !allowUnattended { + if Date().timeIntervalSince(self.lastPromptAt) < 1.0 { + deepLinkLogger.debug("throttling deep link prompt") + return + } + self.lastPromptAt = Date() + + let trimmed = messagePreview.count > 240 ? "\(messagePreview.prefix(240))…" : messagePreview + let body = + "Run the agent with this message?\n\n\(trimmed)\n\nURL:\n\(originalURL.absoluteString)" + guard self.confirm(title: "Run OpenClaw agent?", message: body) else { return } + } + + if AppStateStore.shared.connectionMode == .local { + GatewayProcessManager.shared.setActive(true) + } + + do { + let channel = GatewayAgentChannel(raw: link.channel) + let explicitSessionKey = link.sessionKey? + .trimmingCharacters(in: .whitespacesAndNewlines) + .nonEmpty + let resolvedSessionKey: String = if let explicitSessionKey { + explicitSessionKey + } else { + await GatewayConnection.shared.mainSessionKey() + } + let invocation = GatewayAgentInvocation( + message: messagePreview, + sessionKey: resolvedSessionKey, + thinking: link.thinking?.trimmingCharacters(in: .whitespacesAndNewlines).nonEmpty, + deliver: channel.shouldDeliver(link.deliver), + to: link.to?.trimmingCharacters(in: .whitespacesAndNewlines).nonEmpty, + channel: channel, + timeoutSeconds: link.timeoutSeconds, + idempotencyKey: UUID().uuidString) + + let res = await GatewayConnection.shared.sendAgent(invocation) + if !res.ok { + throw NSError( + domain: "DeepLink", + code: 1, + userInfo: [NSLocalizedDescriptionKey: res.error ?? "agent request failed"]) + } + } catch { + self.presentAlert(title: "Agent request failed", message: error.localizedDescription) + } + } + + // MARK: - Auth + + static func currentKey() -> String { + self.expectedKey() + } + + static func currentCanvasKey() -> String { + self.canvasUnattendedKey + } + + private static func expectedKey() -> String { + let defaults = UserDefaults.standard + if let key = defaults.string(forKey: deepLinkKeyKey), !key.isEmpty { + return key + } + var bytes = [UInt8](repeating: 0, count: 32) + _ = SecRandomCopyBytes(kSecRandomDefault, bytes.count, &bytes) + let data = Data(bytes) + let key = data + .base64EncodedString() + .replacingOccurrences(of: "+", with: "-") + .replacingOccurrences(of: "/", with: "_") + .replacingOccurrences(of: "=", with: "") + defaults.set(key, forKey: deepLinkKeyKey) + return key + } + + private nonisolated static func generateRandomKey() -> String { + var bytes = [UInt8](repeating: 0, count: 32) + _ = SecRandomCopyBytes(kSecRandomDefault, bytes.count, &bytes) + let data = Data(bytes) + return data + .base64EncodedString() + .replacingOccurrences(of: "+", with: "-") + .replacingOccurrences(of: "/", with: "_") + .replacingOccurrences(of: "=", with: "") + } + + // MARK: - UI + + private func confirm(title: String, message: String) -> Bool { + let alert = NSAlert() + alert.messageText = title + alert.informativeText = message + alert.addButton(withTitle: "Run") + alert.addButton(withTitle: "Cancel") + alert.alertStyle = .warning + return alert.runModal() == .alertFirstButtonReturn + } + + private func presentAlert(title: String, message: String) { + let alert = NSAlert() + alert.messageText = title + alert.informativeText = message + alert.addButton(withTitle: "OK") + alert.alertStyle = .informational + alert.runModal() + } +} diff --git a/apps/macos/Sources/OpenClaw/DeviceModelCatalog.swift b/apps/macos/Sources/OpenClaw/DeviceModelCatalog.swift new file mode 100644 index 0000000000000000000000000000000000000000..ce6dd10c9310f3aaa3616c994af9569ef99b5651 --- /dev/null +++ b/apps/macos/Sources/OpenClaw/DeviceModelCatalog.swift @@ -0,0 +1,188 @@ +import Foundation + +struct DevicePresentation: Sendable { + let title: String + let symbol: String? +} + +enum DeviceModelCatalog { + private static let modelIdentifierToName: [String: String] = loadModelIdentifierToName() + private static let resourceBundle: Bundle? = locateResourceBundle() + private static let resourceSubdirectory = "DeviceModels" + + static func presentation(deviceFamily: String?, modelIdentifier: String?) -> DevicePresentation? { + let family = (deviceFamily ?? "").trimmingCharacters(in: .whitespacesAndNewlines) + let model = (modelIdentifier ?? "").trimmingCharacters(in: .whitespacesAndNewlines) + + let friendlyName = model.isEmpty ? nil : self.modelIdentifierToName[model] + let symbol = self.symbol(deviceFamily: family, modelIdentifier: model, friendlyName: friendlyName) + + let title = if let friendlyName, !friendlyName.isEmpty { + friendlyName + } else if !family.isEmpty, !model.isEmpty { + "\(family) (\(model))" + } else if !family.isEmpty { + family + } else if !model.isEmpty { + model + } else { + "" + } + + if title.isEmpty { return nil } + return DevicePresentation(title: title, symbol: symbol) + } + + static func symbol( + deviceFamily familyRaw: String, + modelIdentifier modelIdentifierRaw: String, + friendlyName: String?) -> String? + { + let family = familyRaw.trimmingCharacters(in: .whitespacesAndNewlines) + let modelIdentifier = modelIdentifierRaw.trimmingCharacters(in: .whitespacesAndNewlines) + + return self.symbolFor(modelIdentifier: modelIdentifier, friendlyName: friendlyName) + ?? self.fallbackSymbol(for: family, modelIdentifier: modelIdentifier) + } + + private static func symbolFor(modelIdentifier rawModelIdentifier: String, friendlyName: String?) -> String? { + let modelIdentifier = rawModelIdentifier.trimmingCharacters(in: .whitespacesAndNewlines) + guard !modelIdentifier.isEmpty else { return nil } + + let lower = modelIdentifier.lowercased() + if lower.hasPrefix("ipad") { return "ipad" } + if lower.hasPrefix("iphone") { return "iphone" } + if lower.hasPrefix("ipod") { return "iphone" } + if lower.hasPrefix("watch") { return "applewatch" } + if lower.hasPrefix("appletv") { return "appletv" } + if lower.hasPrefix("audio") || lower.hasPrefix("homepod") { return "speaker" } + + if lower.hasPrefix("macbook") || lower.hasPrefix("macbookpro") || lower.hasPrefix("macbookair") { + return "laptopcomputer" + } + if lower.hasPrefix("macstudio") { return "macstudio" } + if lower.hasPrefix("macmini") { return "macmini" } + if lower.hasPrefix("imac") || lower.hasPrefix("macpro") { return "desktopcomputer" } + + if lower.hasPrefix("mac"), let friendlyNameLower = friendlyName?.lowercased() { + if friendlyNameLower.contains("macbook") { return "laptopcomputer" } + if friendlyNameLower.contains("imac") { return "desktopcomputer" } + if friendlyNameLower.contains("mac mini") { return "macmini" } + if friendlyNameLower.contains("mac studio") { return "macstudio" } + if friendlyNameLower.contains("mac pro") { return "desktopcomputer" } + } + + return nil + } + + private static func fallbackSymbol(for familyRaw: String, modelIdentifier: String) -> String? { + let family = familyRaw.trimmingCharacters(in: .whitespacesAndNewlines) + if family.isEmpty { return nil } + switch family.lowercased() { + case "ipad": + return "ipad" + case "iphone": + return "iphone" + case "mac": + return "laptopcomputer" + case "android": + return "android" + case "linux": + return "cpu" + default: + return "cpu" + } + } + + private static func loadModelIdentifierToName() -> [String: String] { + var combined: [String: String] = [:] + combined.merge( + self.loadMapping(resourceName: "ios-device-identifiers"), + uniquingKeysWith: { current, _ in current }) + combined.merge( + self.loadMapping(resourceName: "mac-device-identifiers"), + uniquingKeysWith: { current, _ in current }) + return combined + } + + private static func loadMapping(resourceName: String) -> [String: String] { + guard let url = self.resourceBundle?.url( + forResource: resourceName, + withExtension: "json", + subdirectory: self.resourceSubdirectory) + else { return [:] } + + do { + let data = try Data(contentsOf: url) + let decoded = try JSONDecoder().decode([String: NameValue].self, from: data) + return decoded.compactMapValues { $0.normalizedName } + } catch { + return [:] + } + } + + private static func locateResourceBundle() -> Bundle? { + // Prefer main bundle (packaged app), then module bundle (SwiftPM/tests). + // Accessing Bundle.module in the packaged app can crash if the bundle isn't where SwiftPM expects it. + if let bundle = self.bundleIfContainsDeviceModels(Bundle.main) { + return bundle + } + + if let bundle = self.bundleIfContainsDeviceModels(Bundle.module) { + return bundle + } + return nil + } + + private static func bundleIfContainsDeviceModels(_ bundle: Bundle) -> Bundle? { + if bundle.url( + forResource: "ios-device-identifiers", + withExtension: "json", + subdirectory: self.resourceSubdirectory) != nil + { + return bundle + } + if bundle.url( + forResource: "mac-device-identifiers", + withExtension: "json", + subdirectory: self.resourceSubdirectory) != nil + { + return bundle + } + return nil + } + + private enum NameValue: Decodable { + case string(String) + case stringArray([String]) + + init(from decoder: Decoder) throws { + let container = try decoder.singleValueContainer() + if let s = try? container.decode(String.self) { + self = .string(s) + return + } + if let arr = try? container.decode([String].self) { + self = .stringArray(arr) + return + } + throw DecodingError.typeMismatch( + String.self, + .init(codingPath: decoder.codingPath, debugDescription: "Expected string or string array")) + } + + var normalizedName: String? { + switch self { + case let .string(s): + let trimmed = s.trimmingCharacters(in: .whitespacesAndNewlines) + return trimmed.isEmpty ? nil : trimmed + case let .stringArray(arr): + let values = arr + .map { $0.trimmingCharacters(in: .whitespacesAndNewlines) } + .filter { !$0.isEmpty } + guard !values.isEmpty else { return nil } + return values.joined(separator: " / ") + } + } + } +} diff --git a/apps/macos/Sources/OpenClaw/DevicePairingApprovalPrompter.swift b/apps/macos/Sources/OpenClaw/DevicePairingApprovalPrompter.swift new file mode 100644 index 0000000000000000000000000000000000000000..73ae0188a39f355308ea49de2ccc8b1958b1f4ba --- /dev/null +++ b/apps/macos/Sources/OpenClaw/DevicePairingApprovalPrompter.swift @@ -0,0 +1,334 @@ +import AppKit +import OpenClawKit +import OpenClawProtocol +import Foundation +import Observation +import OSLog + +@MainActor +@Observable +final class DevicePairingApprovalPrompter { + static let shared = DevicePairingApprovalPrompter() + + private let logger = Logger(subsystem: "ai.openclaw", category: "device-pairing") + private var task: Task? + private var isStopping = false + private var isPresenting = false + private var queue: [PendingRequest] = [] + var pendingCount: Int = 0 + var pendingRepairCount: Int = 0 + private var activeAlert: NSAlert? + private var activeRequestId: String? + private var alertHostWindow: NSWindow? + private var resolvedByRequestId: Set = [] + + private final class AlertHostWindow: NSWindow { + override var canBecomeKey: Bool { true } + override var canBecomeMain: Bool { true } + } + + private struct PairingList: Codable { + let pending: [PendingRequest] + let paired: [PairedDevice]? + } + + private struct PairedDevice: Codable, Equatable { + let deviceId: String + let approvedAtMs: Double? + let displayName: String? + let platform: String? + let remoteIp: String? + } + + private struct PendingRequest: Codable, Equatable, Identifiable { + let requestId: String + let deviceId: String + let publicKey: String + let displayName: String? + let platform: String? + let clientId: String? + let clientMode: String? + let role: String? + let scopes: [String]? + let remoteIp: String? + let silent: Bool? + let isRepair: Bool? + let ts: Double + + var id: String { self.requestId } + } + + private struct PairingResolvedEvent: Codable { + let requestId: String + let deviceId: String + let decision: String + let ts: Double + } + + private enum PairingResolution: String { + case approved + case rejected + } + + func start() { + guard self.task == nil else { return } + self.isStopping = false + self.task = Task { [weak self] in + guard let self else { return } + _ = try? await GatewayConnection.shared.refresh() + await self.loadPendingRequestsFromGateway() + let stream = await GatewayConnection.shared.subscribe(bufferingNewest: 200) + for await push in stream { + if Task.isCancelled { return } + await MainActor.run { [weak self] in self?.handle(push: push) } + } + } + } + + func stop() { + self.isStopping = true + self.endActiveAlert() + self.task?.cancel() + self.task = nil + self.queue.removeAll(keepingCapacity: false) + self.updatePendingCounts() + self.isPresenting = false + self.activeRequestId = nil + self.alertHostWindow?.orderOut(nil) + self.alertHostWindow?.close() + self.alertHostWindow = nil + self.resolvedByRequestId.removeAll(keepingCapacity: false) + } + + private func loadPendingRequestsFromGateway() async { + do { + let list: PairingList = try await GatewayConnection.shared.requestDecoded(method: .devicePairList) + await self.apply(list: list) + } catch { + self.logger.error("failed to load device pairing requests: \(error.localizedDescription, privacy: .public)") + } + } + + private func apply(list: PairingList) async { + self.queue = list.pending.sorted(by: { $0.ts > $1.ts }) + self.updatePendingCounts() + self.presentNextIfNeeded() + } + + private func updatePendingCounts() { + self.pendingCount = self.queue.count + self.pendingRepairCount = self.queue.count(where: { $0.isRepair == true }) + } + + private func presentNextIfNeeded() { + guard !self.isStopping else { return } + guard !self.isPresenting else { return } + guard let next = self.queue.first else { return } + self.isPresenting = true + self.presentAlert(for: next) + } + + private func presentAlert(for req: PendingRequest) { + self.logger.info("presenting device pairing alert requestId=\(req.requestId, privacy: .public)") + NSApp.activate(ignoringOtherApps: true) + + let alert = NSAlert() + alert.alertStyle = .warning + alert.messageText = "Allow device to connect?" + alert.informativeText = Self.describe(req) + alert.addButton(withTitle: "Later") + alert.addButton(withTitle: "Approve") + alert.addButton(withTitle: "Reject") + if #available(macOS 11.0, *), alert.buttons.indices.contains(2) { + alert.buttons[2].hasDestructiveAction = true + } + + self.activeAlert = alert + self.activeRequestId = req.requestId + let hostWindow = self.requireAlertHostWindow() + + let sheetSize = alert.window.frame.size + if let screen = hostWindow.screen ?? NSScreen.main { + let bounds = screen.visibleFrame + let x = bounds.midX - (sheetSize.width / 2) + let sheetOriginY = bounds.midY - (sheetSize.height / 2) + let hostY = sheetOriginY + sheetSize.height - hostWindow.frame.height + hostWindow.setFrameOrigin(NSPoint(x: x, y: hostY)) + } else { + hostWindow.center() + } + + hostWindow.makeKeyAndOrderFront(nil) + alert.beginSheetModal(for: hostWindow) { [weak self] response in + Task { @MainActor [weak self] in + guard let self else { return } + self.activeRequestId = nil + self.activeAlert = nil + await self.handleAlertResponse(response, request: req) + hostWindow.orderOut(nil) + } + } + } + + private func handleAlertResponse(_ response: NSApplication.ModalResponse, request: PendingRequest) async { + var shouldRemove = response != .alertFirstButtonReturn + defer { + if shouldRemove { + if self.queue.first == request { + self.queue.removeFirst() + } else { + self.queue.removeAll { $0 == request } + } + } + self.updatePendingCounts() + self.isPresenting = false + self.presentNextIfNeeded() + } + + guard !self.isStopping else { return } + + if self.resolvedByRequestId.remove(request.requestId) != nil { + return + } + + switch response { + case .alertFirstButtonReturn: + shouldRemove = false + if let idx = self.queue.firstIndex(of: request) { + self.queue.remove(at: idx) + } + self.queue.append(request) + return + case .alertSecondButtonReturn: + _ = await self.approve(requestId: request.requestId) + case .alertThirdButtonReturn: + await self.reject(requestId: request.requestId) + default: + return + } + } + + private func approve(requestId: String) async -> Bool { + do { + try await GatewayConnection.shared.devicePairApprove(requestId: requestId) + self.logger.info("approved device pairing requestId=\(requestId, privacy: .public)") + return true + } catch { + self.logger.error("approve failed requestId=\(requestId, privacy: .public)") + self.logger.error("approve failed: \(error.localizedDescription, privacy: .public)") + return false + } + } + + private func reject(requestId: String) async { + do { + try await GatewayConnection.shared.devicePairReject(requestId: requestId) + self.logger.info("rejected device pairing requestId=\(requestId, privacy: .public)") + } catch { + self.logger.error("reject failed requestId=\(requestId, privacy: .public)") + self.logger.error("reject failed: \(error.localizedDescription, privacy: .public)") + } + } + + private func endActiveAlert() { + guard let alert = self.activeAlert else { return } + if let parent = alert.window.sheetParent { + parent.endSheet(alert.window, returnCode: .abort) + } + self.activeAlert = nil + self.activeRequestId = nil + } + + private func requireAlertHostWindow() -> NSWindow { + if let alertHostWindow { + return alertHostWindow + } + + let window = AlertHostWindow( + contentRect: NSRect(x: 0, y: 0, width: 520, height: 1), + styleMask: [.borderless], + backing: .buffered, + defer: false) + window.title = "" + window.isReleasedWhenClosed = false + window.level = .floating + window.collectionBehavior = [.canJoinAllSpaces, .fullScreenAuxiliary] + window.isOpaque = false + window.hasShadow = false + window.backgroundColor = .clear + window.ignoresMouseEvents = true + + self.alertHostWindow = window + return window + } + + private func handle(push: GatewayPush) { + switch push { + case let .event(evt) where evt.event == "device.pair.requested": + guard let payload = evt.payload else { return } + do { + let req = try GatewayPayloadDecoding.decode(payload, as: PendingRequest.self) + self.enqueue(req) + } catch { + self.logger + .error("failed to decode device pairing request: \(error.localizedDescription, privacy: .public)") + } + case let .event(evt) where evt.event == "device.pair.resolved": + guard let payload = evt.payload else { return } + do { + let resolved = try GatewayPayloadDecoding.decode(payload, as: PairingResolvedEvent.self) + self.handleResolved(resolved) + } catch { + self.logger + .error( + "failed to decode device pairing resolution: \(error.localizedDescription, privacy: .public)") + } + default: + break + } + } + + private func enqueue(_ req: PendingRequest) { + guard !self.queue.contains(req) else { return } + self.queue.append(req) + self.updatePendingCounts() + self.presentNextIfNeeded() + } + + private func handleResolved(_ resolved: PairingResolvedEvent) { + let resolution = resolved.decision == PairingResolution.approved.rawValue ? PairingResolution + .approved : .rejected + if let activeRequestId, activeRequestId == resolved.requestId { + self.resolvedByRequestId.insert(resolved.requestId) + self.endActiveAlert() + let decision = resolution.rawValue + self.logger.info( + "device pairing resolved while active requestId=\(resolved.requestId, privacy: .public) " + + "decision=\(decision, privacy: .public)") + return + } + self.queue.removeAll { $0.requestId == resolved.requestId } + self.updatePendingCounts() + } + + private static func describe(_ req: PendingRequest) -> String { + var lines: [String] = [] + lines.append("Device: \(req.displayName ?? req.deviceId)") + if let platform = req.platform { + lines.append("Platform: \(platform)") + } + if let role = req.role { + lines.append("Role: \(role)") + } + if let scopes = req.scopes, !scopes.isEmpty { + lines.append("Scopes: \(scopes.joined(separator: ", "))") + } + if let remoteIp = req.remoteIp { + lines.append("IP: \(remoteIp)") + } + if req.isRepair == true { + lines.append("Repair: yes") + } + return lines.joined(separator: "\n") + } +} diff --git a/apps/macos/Sources/OpenClaw/DiagnosticsFileLog.swift b/apps/macos/Sources/OpenClaw/DiagnosticsFileLog.swift new file mode 100644 index 0000000000000000000000000000000000000000..44baa738bdc23ecd7edd3cc755d6aa7f3f8702a0 --- /dev/null +++ b/apps/macos/Sources/OpenClaw/DiagnosticsFileLog.swift @@ -0,0 +1,133 @@ +import Foundation + +actor DiagnosticsFileLog { + static let shared = DiagnosticsFileLog() + + private let fileName = "diagnostics.jsonl" + private let maxBytes: Int64 = 5 * 1024 * 1024 + private let maxBackups = 5 + + struct Record: Codable, Sendable { + let ts: String + let pid: Int32 + let category: String + let event: String + let fields: [String: String]? + } + + nonisolated static func isEnabled() -> Bool { + UserDefaults.standard.bool(forKey: debugFileLogEnabledKey) + } + + nonisolated static func logDirectoryURL() -> URL { + let library = FileManager().urls(for: .libraryDirectory, in: .userDomainMask).first + ?? FileManager().homeDirectoryForCurrentUser.appendingPathComponent("Library", isDirectory: true) + return library + .appendingPathComponent("Logs", isDirectory: true) + .appendingPathComponent("OpenClaw", isDirectory: true) + } + + nonisolated static func logFileURL() -> URL { + self.logDirectoryURL().appendingPathComponent("diagnostics.jsonl", isDirectory: false) + } + + nonisolated func log(category: String, event: String, fields: [String: String]? = nil) { + guard Self.isEnabled() else { return } + let record = Record( + ts: ISO8601DateFormatter().string(from: Date()), + pid: ProcessInfo.processInfo.processIdentifier, + category: category, + event: event, + fields: fields) + Task { await self.write(record: record) } + } + + func clear() throws { + let fm = FileManager() + let base = Self.logFileURL() + if fm.fileExists(atPath: base.path) { + try fm.removeItem(at: base) + } + for idx in 1...self.maxBackups { + let url = self.rotatedURL(index: idx) + if fm.fileExists(atPath: url.path) { + try fm.removeItem(at: url) + } + } + } + + private func write(record: Record) { + do { + try self.ensureDirectory() + try self.rotateIfNeeded() + try self.append(record: record) + } catch { + // Best-effort only: never crash or block the app on logging. + } + } + + private func ensureDirectory() throws { + try FileManager().createDirectory( + at: Self.logDirectoryURL(), + withIntermediateDirectories: true) + } + + private func append(record: Record) throws { + let url = Self.logFileURL() + let data = try JSONEncoder().encode(record) + var line = Data() + line.append(data) + line.append(0x0A) // newline + + let fm = FileManager() + if !fm.fileExists(atPath: url.path) { + fm.createFile(atPath: url.path, contents: nil) + } + + let handle = try FileHandle(forWritingTo: url) + defer { try? handle.close() } + try handle.seekToEnd() + try handle.write(contentsOf: line) + } + + private func rotateIfNeeded() throws { + let url = Self.logFileURL() + guard let attrs = try? FileManager().attributesOfItem(atPath: url.path), + let size = attrs[.size] as? NSNumber + else { return } + + if size.int64Value < self.maxBytes { return } + + let fm = FileManager() + + let oldest = self.rotatedURL(index: self.maxBackups) + if fm.fileExists(atPath: oldest.path) { + try fm.removeItem(at: oldest) + } + + if self.maxBackups > 1 { + for idx in stride(from: self.maxBackups - 1, through: 1, by: -1) { + let src = self.rotatedURL(index: idx) + let dst = self.rotatedURL(index: idx + 1) + if fm.fileExists(atPath: src.path) { + if fm.fileExists(atPath: dst.path) { + try fm.removeItem(at: dst) + } + try fm.moveItem(at: src, to: dst) + } + } + } + + let first = self.rotatedURL(index: 1) + if fm.fileExists(atPath: first.path) { + try fm.removeItem(at: first) + } + if fm.fileExists(atPath: url.path) { + try fm.moveItem(at: url, to: first) + } + } + + private func rotatedURL(index: Int) -> URL { + Self.logDirectoryURL().appendingPathComponent("\(self.fileName).\(index)", isDirectory: false) + } +} diff --git a/apps/macos/Sources/OpenClaw/DockIconManager.swift b/apps/macos/Sources/OpenClaw/DockIconManager.swift new file mode 100644 index 0000000000000000000000000000000000000000..98201393b7553ac7dce90e1d0ad5fdee526fbffc --- /dev/null +++ b/apps/macos/Sources/OpenClaw/DockIconManager.swift @@ -0,0 +1,116 @@ +import AppKit + +/// Central manager for Dock icon visibility. +/// Shows the Dock icon while any windows are visible, regardless of user preference. +final class DockIconManager: NSObject, @unchecked Sendable { + static let shared = DockIconManager() + + private var windowsObservation: NSKeyValueObservation? + private let logger = Logger(subsystem: "ai.openclaw", category: "DockIconManager") + + override private init() { + super.init() + self.setupObservers() + Task { @MainActor in + self.updateDockVisibility() + } + } + + deinit { + self.windowsObservation?.invalidate() + NotificationCenter.default.removeObserver(self) + } + + func updateDockVisibility() { + Task { @MainActor in + guard NSApp != nil else { + self.logger.warning("NSApp not ready, skipping Dock visibility update") + return + } + + let userWantsDockHidden = !UserDefaults.standard.bool(forKey: showDockIconKey) + let visibleWindows = NSApp?.windows.filter { window in + window.isVisible && + window.frame.width > 1 && + window.frame.height > 1 && + !window.isKind(of: NSPanel.self) && + "\(type(of: window))" != "NSPopupMenuWindow" && + window.contentViewController != nil + } ?? [] + + let hasVisibleWindows = !visibleWindows.isEmpty + if !userWantsDockHidden || hasVisibleWindows { + NSApp?.setActivationPolicy(.regular) + } else { + NSApp?.setActivationPolicy(.accessory) + } + } + } + + func temporarilyShowDock() { + Task { @MainActor in + guard NSApp != nil else { + self.logger.warning("NSApp not ready, cannot show Dock icon") + return + } + NSApp.setActivationPolicy(.regular) + } + } + + private func setupObservers() { + Task { @MainActor in + guard let app = NSApp else { + self.logger.warning("NSApp not ready, delaying Dock observers") + try? await Task.sleep(for: .milliseconds(200)) + self.setupObservers() + return + } + + self.windowsObservation = app.observe(\.windows, options: [.new]) { [weak self] _, _ in + Task { @MainActor in + try? await Task.sleep(for: .milliseconds(50)) + self?.updateDockVisibility() + } + } + + NotificationCenter.default.addObserver( + self, + selector: #selector(self.windowVisibilityChanged), + name: NSWindow.didBecomeKeyNotification, + object: nil) + NotificationCenter.default.addObserver( + self, + selector: #selector(self.windowVisibilityChanged), + name: NSWindow.didResignKeyNotification, + object: nil) + NotificationCenter.default.addObserver( + self, + selector: #selector(self.windowVisibilityChanged), + name: NSWindow.willCloseNotification, + object: nil) + NotificationCenter.default.addObserver( + self, + selector: #selector(self.dockPreferenceChanged), + name: UserDefaults.didChangeNotification, + object: nil) + } + } + + @objc + private func windowVisibilityChanged(_: Notification) { + Task { @MainActor in + self.updateDockVisibility() + } + } + + @objc + private func dockPreferenceChanged(_ notification: Notification) { + guard let userDefaults = notification.object as? UserDefaults, + userDefaults == UserDefaults.standard + else { return } + + Task { @MainActor in + self.updateDockVisibility() + } + } +} diff --git a/apps/macos/Sources/OpenClaw/ExecApprovals.swift b/apps/macos/Sources/OpenClaw/ExecApprovals.swift new file mode 100644 index 0000000000000000000000000000000000000000..21ab5b1749f53f60f2800a433a66fee120ccc52e --- /dev/null +++ b/apps/macos/Sources/OpenClaw/ExecApprovals.swift @@ -0,0 +1,790 @@ +import CryptoKit +import Foundation +import OSLog +import Security + +enum ExecSecurity: String, CaseIterable, Codable, Identifiable { + case deny + case allowlist + case full + + var id: String { self.rawValue } + + var title: String { + switch self { + case .deny: "Deny" + case .allowlist: "Allowlist" + case .full: "Always Allow" + } + } +} + +enum ExecApprovalQuickMode: String, CaseIterable, Identifiable { + case deny + case ask + case allow + + var id: String { self.rawValue } + + var title: String { + switch self { + case .deny: "Deny" + case .ask: "Always Ask" + case .allow: "Always Allow" + } + } + + var security: ExecSecurity { + switch self { + case .deny: .deny + case .ask: .allowlist + case .allow: .full + } + } + + var ask: ExecAsk { + switch self { + case .deny: .off + case .ask: .onMiss + case .allow: .off + } + } + + static func from(security: ExecSecurity, ask: ExecAsk) -> ExecApprovalQuickMode { + switch security { + case .deny: + .deny + case .full: + .allow + case .allowlist: + .ask + } + } +} + +enum ExecAsk: String, CaseIterable, Codable, Identifiable { + case off + case onMiss = "on-miss" + case always + + var id: String { self.rawValue } + + var title: String { + switch self { + case .off: "Never Ask" + case .onMiss: "Ask on Allowlist Miss" + case .always: "Always Ask" + } + } +} + +enum ExecApprovalDecision: String, Codable, Sendable { + case allowOnce = "allow-once" + case allowAlways = "allow-always" + case deny +} + +struct ExecAllowlistEntry: Codable, Hashable, Identifiable { + var id: UUID + var pattern: String + var lastUsedAt: Double? + var lastUsedCommand: String? + var lastResolvedPath: String? + + init( + id: UUID = UUID(), + pattern: String, + lastUsedAt: Double? = nil, + lastUsedCommand: String? = nil, + lastResolvedPath: String? = nil) + { + self.id = id + self.pattern = pattern + self.lastUsedAt = lastUsedAt + self.lastUsedCommand = lastUsedCommand + self.lastResolvedPath = lastResolvedPath + } + + private enum CodingKeys: String, CodingKey { + case id + case pattern + case lastUsedAt + case lastUsedCommand + case lastResolvedPath + } + + init(from decoder: Decoder) throws { + let container = try decoder.container(keyedBy: CodingKeys.self) + self.id = try container.decodeIfPresent(UUID.self, forKey: .id) ?? UUID() + self.pattern = try container.decode(String.self, forKey: .pattern) + self.lastUsedAt = try container.decodeIfPresent(Double.self, forKey: .lastUsedAt) + self.lastUsedCommand = try container.decodeIfPresent(String.self, forKey: .lastUsedCommand) + self.lastResolvedPath = try container.decodeIfPresent(String.self, forKey: .lastResolvedPath) + } + + func encode(to encoder: Encoder) throws { + var container = encoder.container(keyedBy: CodingKeys.self) + try container.encode(self.id, forKey: .id) + try container.encode(self.pattern, forKey: .pattern) + try container.encodeIfPresent(self.lastUsedAt, forKey: .lastUsedAt) + try container.encodeIfPresent(self.lastUsedCommand, forKey: .lastUsedCommand) + try container.encodeIfPresent(self.lastResolvedPath, forKey: .lastResolvedPath) + } +} + +struct ExecApprovalsDefaults: Codable { + var security: ExecSecurity? + var ask: ExecAsk? + var askFallback: ExecSecurity? + var autoAllowSkills: Bool? +} + +struct ExecApprovalsAgent: Codable { + var security: ExecSecurity? + var ask: ExecAsk? + var askFallback: ExecSecurity? + var autoAllowSkills: Bool? + var allowlist: [ExecAllowlistEntry]? + + var isEmpty: Bool { + self.security == nil && self.ask == nil && self.askFallback == nil && self + .autoAllowSkills == nil && (self.allowlist?.isEmpty ?? true) + } +} + +struct ExecApprovalsSocketConfig: Codable { + var path: String? + var token: String? +} + +struct ExecApprovalsFile: Codable { + var version: Int + var socket: ExecApprovalsSocketConfig? + var defaults: ExecApprovalsDefaults? + var agents: [String: ExecApprovalsAgent]? +} + +struct ExecApprovalsSnapshot: Codable { + var path: String + var exists: Bool + var hash: String + var file: ExecApprovalsFile +} + +struct ExecApprovalsResolved { + let url: URL + let socketPath: String + let token: String + let defaults: ExecApprovalsResolvedDefaults + let agent: ExecApprovalsResolvedDefaults + let allowlist: [ExecAllowlistEntry] + var file: ExecApprovalsFile +} + +struct ExecApprovalsResolvedDefaults { + var security: ExecSecurity + var ask: ExecAsk + var askFallback: ExecSecurity + var autoAllowSkills: Bool +} + +enum ExecApprovalsStore { + private static let logger = Logger(subsystem: "ai.openclaw", category: "exec-approvals") + private static let defaultAgentId = "main" + private static let defaultSecurity: ExecSecurity = .deny + private static let defaultAsk: ExecAsk = .onMiss + private static let defaultAskFallback: ExecSecurity = .deny + private static let defaultAutoAllowSkills = false + + static func fileURL() -> URL { + OpenClawPaths.stateDirURL.appendingPathComponent("exec-approvals.json") + } + + static func socketPath() -> String { + OpenClawPaths.stateDirURL.appendingPathComponent("exec-approvals.sock").path + } + + static func normalizeIncoming(_ file: ExecApprovalsFile) -> ExecApprovalsFile { + let socketPath = file.socket?.path?.trimmingCharacters(in: .whitespacesAndNewlines) ?? "" + let token = file.socket?.token?.trimmingCharacters(in: .whitespacesAndNewlines) ?? "" + var agents = file.agents ?? [:] + if let legacyDefault = agents["default"] { + if let main = agents[self.defaultAgentId] { + agents[self.defaultAgentId] = self.mergeAgents(current: main, legacy: legacyDefault) + } else { + agents[self.defaultAgentId] = legacyDefault + } + agents.removeValue(forKey: "default") + } + return ExecApprovalsFile( + version: 1, + socket: ExecApprovalsSocketConfig( + path: socketPath.isEmpty ? nil : socketPath, + token: token.isEmpty ? nil : token), + defaults: file.defaults, + agents: agents) + } + + static func readSnapshot() -> ExecApprovalsSnapshot { + let url = self.fileURL() + guard FileManager().fileExists(atPath: url.path) else { + return ExecApprovalsSnapshot( + path: url.path, + exists: false, + hash: self.hashRaw(nil), + file: ExecApprovalsFile(version: 1, socket: nil, defaults: nil, agents: [:])) + } + let raw = try? String(contentsOf: url, encoding: .utf8) + let data = raw.flatMap { $0.data(using: .utf8) } + let decoded: ExecApprovalsFile = { + if let data, let file = try? JSONDecoder().decode(ExecApprovalsFile.self, from: data), file.version == 1 { + return file + } + return ExecApprovalsFile(version: 1, socket: nil, defaults: nil, agents: [:]) + }() + return ExecApprovalsSnapshot( + path: url.path, + exists: true, + hash: self.hashRaw(raw), + file: decoded) + } + + static func redactForSnapshot(_ file: ExecApprovalsFile) -> ExecApprovalsFile { + let socketPath = file.socket?.path?.trimmingCharacters(in: .whitespacesAndNewlines) ?? "" + if socketPath.isEmpty { + return ExecApprovalsFile( + version: file.version, + socket: nil, + defaults: file.defaults, + agents: file.agents) + } + return ExecApprovalsFile( + version: file.version, + socket: ExecApprovalsSocketConfig(path: socketPath, token: nil), + defaults: file.defaults, + agents: file.agents) + } + + static func loadFile() -> ExecApprovalsFile { + let url = self.fileURL() + guard FileManager().fileExists(atPath: url.path) else { + return ExecApprovalsFile(version: 1, socket: nil, defaults: nil, agents: [:]) + } + do { + let data = try Data(contentsOf: url) + let decoded = try JSONDecoder().decode(ExecApprovalsFile.self, from: data) + if decoded.version != 1 { + return ExecApprovalsFile(version: 1, socket: nil, defaults: nil, agents: [:]) + } + return decoded + } catch { + self.logger.warning("exec approvals load failed: \(error.localizedDescription, privacy: .public)") + return ExecApprovalsFile(version: 1, socket: nil, defaults: nil, agents: [:]) + } + } + + static func saveFile(_ file: ExecApprovalsFile) { + do { + let encoder = JSONEncoder() + encoder.outputFormatting = [.prettyPrinted, .sortedKeys] + let data = try encoder.encode(file) + let url = self.fileURL() + try FileManager().createDirectory( + at: url.deletingLastPathComponent(), + withIntermediateDirectories: true) + try data.write(to: url, options: [.atomic]) + try? FileManager().setAttributes([.posixPermissions: 0o600], ofItemAtPath: url.path) + } catch { + self.logger.error("exec approvals save failed: \(error.localizedDescription, privacy: .public)") + } + } + + static func ensureFile() -> ExecApprovalsFile { + var file = self.loadFile() + if file.socket == nil { file.socket = ExecApprovalsSocketConfig(path: nil, token: nil) } + let path = file.socket?.path?.trimmingCharacters(in: .whitespacesAndNewlines) ?? "" + if path.isEmpty { + file.socket?.path = self.socketPath() + } + let token = file.socket?.token?.trimmingCharacters(in: .whitespacesAndNewlines) ?? "" + if token.isEmpty { + file.socket?.token = self.generateToken() + } + if file.agents == nil { file.agents = [:] } + self.saveFile(file) + return file + } + + static func resolve(agentId: String?) -> ExecApprovalsResolved { + let file = self.ensureFile() + let defaults = file.defaults ?? ExecApprovalsDefaults() + let resolvedDefaults = ExecApprovalsResolvedDefaults( + security: defaults.security ?? self.defaultSecurity, + ask: defaults.ask ?? self.defaultAsk, + askFallback: defaults.askFallback ?? self.defaultAskFallback, + autoAllowSkills: defaults.autoAllowSkills ?? self.defaultAutoAllowSkills) + let key = self.agentKey(agentId) + let agentEntry = file.agents?[key] ?? ExecApprovalsAgent() + let wildcardEntry = file.agents?["*"] ?? ExecApprovalsAgent() + let resolvedAgent = ExecApprovalsResolvedDefaults( + security: agentEntry.security ?? wildcardEntry.security ?? resolvedDefaults.security, + ask: agentEntry.ask ?? wildcardEntry.ask ?? resolvedDefaults.ask, + askFallback: agentEntry.askFallback ?? wildcardEntry.askFallback + ?? resolvedDefaults.askFallback, + autoAllowSkills: agentEntry.autoAllowSkills ?? wildcardEntry.autoAllowSkills + ?? resolvedDefaults.autoAllowSkills) + let allowlist = ((wildcardEntry.allowlist ?? []) + (agentEntry.allowlist ?? [])) + .map { entry in + ExecAllowlistEntry( + id: entry.id, + pattern: entry.pattern.trimmingCharacters(in: .whitespacesAndNewlines), + lastUsedAt: entry.lastUsedAt, + lastUsedCommand: entry.lastUsedCommand, + lastResolvedPath: entry.lastResolvedPath) + } + .filter { !$0.pattern.isEmpty } + let socketPath = self.expandPath(file.socket?.path ?? self.socketPath()) + let token = file.socket?.token ?? "" + return ExecApprovalsResolved( + url: self.fileURL(), + socketPath: socketPath, + token: token, + defaults: resolvedDefaults, + agent: resolvedAgent, + allowlist: allowlist, + file: file) + } + + static func resolveDefaults() -> ExecApprovalsResolvedDefaults { + let file = self.ensureFile() + let defaults = file.defaults ?? ExecApprovalsDefaults() + return ExecApprovalsResolvedDefaults( + security: defaults.security ?? self.defaultSecurity, + ask: defaults.ask ?? self.defaultAsk, + askFallback: defaults.askFallback ?? self.defaultAskFallback, + autoAllowSkills: defaults.autoAllowSkills ?? self.defaultAutoAllowSkills) + } + + static func saveDefaults(_ defaults: ExecApprovalsDefaults) { + self.updateFile { file in + file.defaults = defaults + } + } + + static func updateDefaults(_ mutate: (inout ExecApprovalsDefaults) -> Void) { + self.updateFile { file in + var defaults = file.defaults ?? ExecApprovalsDefaults() + mutate(&defaults) + file.defaults = defaults + } + } + + static func saveAgent(_ agent: ExecApprovalsAgent, agentId: String?) { + self.updateFile { file in + var agents = file.agents ?? [:] + let key = self.agentKey(agentId) + if agent.isEmpty { + agents.removeValue(forKey: key) + } else { + agents[key] = agent + } + file.agents = agents.isEmpty ? nil : agents + } + } + + static func addAllowlistEntry(agentId: String?, pattern: String) { + let trimmed = pattern.trimmingCharacters(in: .whitespacesAndNewlines) + guard !trimmed.isEmpty else { return } + self.updateFile { file in + let key = self.agentKey(agentId) + var agents = file.agents ?? [:] + var entry = agents[key] ?? ExecApprovalsAgent() + var allowlist = entry.allowlist ?? [] + if allowlist.contains(where: { $0.pattern == trimmed }) { return } + allowlist.append(ExecAllowlistEntry(pattern: trimmed, lastUsedAt: Date().timeIntervalSince1970 * 1000)) + entry.allowlist = allowlist + agents[key] = entry + file.agents = agents + } + } + + static func recordAllowlistUse( + agentId: String?, + pattern: String, + command: String, + resolvedPath: String?) + { + self.updateFile { file in + let key = self.agentKey(agentId) + var agents = file.agents ?? [:] + var entry = agents[key] ?? ExecApprovalsAgent() + let allowlist = (entry.allowlist ?? []).map { item -> ExecAllowlistEntry in + guard item.pattern == pattern else { return item } + return ExecAllowlistEntry( + id: item.id, + pattern: item.pattern, + lastUsedAt: Date().timeIntervalSince1970 * 1000, + lastUsedCommand: command, + lastResolvedPath: resolvedPath) + } + entry.allowlist = allowlist + agents[key] = entry + file.agents = agents + } + } + + static func updateAllowlist(agentId: String?, allowlist: [ExecAllowlistEntry]) { + self.updateFile { file in + let key = self.agentKey(agentId) + var agents = file.agents ?? [:] + var entry = agents[key] ?? ExecApprovalsAgent() + let cleaned = allowlist + .map { item in + ExecAllowlistEntry( + id: item.id, + pattern: item.pattern.trimmingCharacters(in: .whitespacesAndNewlines), + lastUsedAt: item.lastUsedAt, + lastUsedCommand: item.lastUsedCommand, + lastResolvedPath: item.lastResolvedPath) + } + .filter { !$0.pattern.isEmpty } + entry.allowlist = cleaned + agents[key] = entry + file.agents = agents + } + } + + static func updateAgentSettings(agentId: String?, mutate: (inout ExecApprovalsAgent) -> Void) { + self.updateFile { file in + let key = self.agentKey(agentId) + var agents = file.agents ?? [:] + var entry = agents[key] ?? ExecApprovalsAgent() + mutate(&entry) + if entry.isEmpty { + agents.removeValue(forKey: key) + } else { + agents[key] = entry + } + file.agents = agents.isEmpty ? nil : agents + } + } + + private static func updateFile(_ mutate: (inout ExecApprovalsFile) -> Void) { + var file = self.ensureFile() + mutate(&file) + self.saveFile(file) + } + + private static func generateToken() -> String { + var bytes = [UInt8](repeating: 0, count: 24) + let status = SecRandomCopyBytes(kSecRandomDefault, bytes.count, &bytes) + if status == errSecSuccess { + return Data(bytes) + .base64EncodedString() + .replacingOccurrences(of: "+", with: "-") + .replacingOccurrences(of: "/", with: "_") + .replacingOccurrences(of: "=", with: "") + } + return UUID().uuidString + } + + private static func hashRaw(_ raw: String?) -> String { + let data = Data((raw ?? "").utf8) + let digest = SHA256.hash(data: data) + return digest.map { String(format: "%02x", $0) }.joined() + } + + private static func expandPath(_ raw: String) -> String { + let trimmed = raw.trimmingCharacters(in: .whitespacesAndNewlines) + if trimmed == "~" { + return FileManager().homeDirectoryForCurrentUser.path + } + if trimmed.hasPrefix("~/") { + let suffix = trimmed.dropFirst(2) + return FileManager().homeDirectoryForCurrentUser + .appendingPathComponent(String(suffix)).path + } + return trimmed + } + + private static func agentKey(_ agentId: String?) -> String { + let trimmed = agentId?.trimmingCharacters(in: .whitespacesAndNewlines) ?? "" + return trimmed.isEmpty ? self.defaultAgentId : trimmed + } + + private static func normalizedPattern(_ pattern: String?) -> String? { + let trimmed = pattern?.trimmingCharacters(in: .whitespacesAndNewlines) ?? "" + return trimmed.isEmpty ? nil : trimmed.lowercased() + } + + private static func mergeAgents( + current: ExecApprovalsAgent, + legacy: ExecApprovalsAgent) -> ExecApprovalsAgent + { + var seen = Set() + var allowlist: [ExecAllowlistEntry] = [] + func append(_ entry: ExecAllowlistEntry) { + guard let key = self.normalizedPattern(entry.pattern), !seen.contains(key) else { + return + } + seen.insert(key) + allowlist.append(entry) + } + for entry in current.allowlist ?? [] { + append(entry) + } + for entry in legacy.allowlist ?? [] { + append(entry) + } + + return ExecApprovalsAgent( + security: current.security ?? legacy.security, + ask: current.ask ?? legacy.ask, + askFallback: current.askFallback ?? legacy.askFallback, + autoAllowSkills: current.autoAllowSkills ?? legacy.autoAllowSkills, + allowlist: allowlist.isEmpty ? nil : allowlist) + } +} + +struct ExecCommandResolution: Sendable { + let rawExecutable: String + let resolvedPath: String? + let executableName: String + let cwd: String? + + static func resolve( + command: [String], + rawCommand: String?, + cwd: String?, + env: [String: String]?) -> ExecCommandResolution? + { + let trimmedRaw = rawCommand?.trimmingCharacters(in: .whitespacesAndNewlines) ?? "" + if !trimmedRaw.isEmpty, let token = self.parseFirstToken(trimmedRaw) { + return self.resolveExecutable(rawExecutable: token, cwd: cwd, env: env) + } + return self.resolve(command: command, cwd: cwd, env: env) + } + + static func resolve(command: [String], cwd: String?, env: [String: String]?) -> ExecCommandResolution? { + guard let raw = command.first?.trimmingCharacters(in: .whitespacesAndNewlines), !raw.isEmpty else { + return nil + } + return self.resolveExecutable(rawExecutable: raw, cwd: cwd, env: env) + } + + private static func resolveExecutable( + rawExecutable: String, + cwd: String?, + env: [String: String]?) -> ExecCommandResolution? + { + let expanded = rawExecutable.hasPrefix("~") ? (rawExecutable as NSString).expandingTildeInPath : rawExecutable + let hasPathSeparator = expanded.contains("/") || expanded.contains("\\") + let resolvedPath: String? = { + if hasPathSeparator { + if expanded.hasPrefix("/") { + return expanded + } + let base = cwd?.trimmingCharacters(in: .whitespacesAndNewlines) + let root = (base?.isEmpty == false) ? base! : FileManager().currentDirectoryPath + return URL(fileURLWithPath: root).appendingPathComponent(expanded).path + } + let searchPaths = self.searchPaths(from: env) + return CommandResolver.findExecutable(named: expanded, searchPaths: searchPaths) + }() + let name = resolvedPath.map { URL(fileURLWithPath: $0).lastPathComponent } ?? expanded + return ExecCommandResolution( + rawExecutable: expanded, + resolvedPath: resolvedPath, + executableName: name, + cwd: cwd) + } + + private static func parseFirstToken(_ command: String) -> String? { + let trimmed = command.trimmingCharacters(in: .whitespacesAndNewlines) + guard !trimmed.isEmpty else { return nil } + guard let first = trimmed.first else { return nil } + if first == "\"" || first == "'" { + let rest = trimmed.dropFirst() + if let end = rest.firstIndex(of: first) { + return String(rest[.. [String] { + let raw = env?["PATH"] + if let raw, !raw.isEmpty { + return raw.split(separator: ":").map(String.init) + } + return CommandResolver.preferredPaths() + } +} + +enum ExecCommandFormatter { + static func displayString(for argv: [String]) -> String { + argv.map { arg in + let trimmed = arg.trimmingCharacters(in: .whitespacesAndNewlines) + guard !trimmed.isEmpty else { return "\"\"" } + let needsQuotes = trimmed.contains { $0.isWhitespace || $0 == "\"" } + if !needsQuotes { return trimmed } + let escaped = trimmed.replacingOccurrences(of: "\"", with: "\\\"") + return "\"\(escaped)\"" + }.joined(separator: " ") + } + + static func displayString(for argv: [String], rawCommand: String?) -> String { + let trimmed = rawCommand?.trimmingCharacters(in: .whitespacesAndNewlines) ?? "" + if !trimmed.isEmpty { return trimmed } + return self.displayString(for: argv) + } +} + +enum ExecApprovalHelpers { + static func parseDecision(_ raw: String?) -> ExecApprovalDecision? { + let trimmed = raw?.trimmingCharacters(in: .whitespacesAndNewlines) ?? "" + guard !trimmed.isEmpty else { return nil } + return ExecApprovalDecision(rawValue: trimmed) + } + + static func requiresAsk( + ask: ExecAsk, + security: ExecSecurity, + allowlistMatch: ExecAllowlistEntry?, + skillAllow: Bool) -> Bool + { + if ask == .always { return true } + if ask == .onMiss, security == .allowlist, allowlistMatch == nil, !skillAllow { return true } + return false + } + + static func allowlistPattern(command: [String], resolution: ExecCommandResolution?) -> String? { + let pattern = resolution?.resolvedPath ?? resolution?.rawExecutable ?? command.first ?? "" + return pattern.isEmpty ? nil : pattern + } +} + +enum ExecAllowlistMatcher { + static func match(entries: [ExecAllowlistEntry], resolution: ExecCommandResolution?) -> ExecAllowlistEntry? { + guard let resolution, !entries.isEmpty else { return nil } + let rawExecutable = resolution.rawExecutable + let resolvedPath = resolution.resolvedPath + let executableName = resolution.executableName + + for entry in entries { + let pattern = entry.pattern.trimmingCharacters(in: .whitespacesAndNewlines) + if pattern.isEmpty { continue } + let hasPath = pattern.contains("/") || pattern.contains("~") || pattern.contains("\\") + if hasPath { + let target = resolvedPath ?? rawExecutable + if self.matches(pattern: pattern, target: target) { return entry } + } else if self.matches(pattern: pattern, target: executableName) { + return entry + } + } + return nil + } + + private static func matches(pattern: String, target: String) -> Bool { + let trimmed = pattern.trimmingCharacters(in: .whitespacesAndNewlines) + guard !trimmed.isEmpty else { return false } + let expanded = trimmed.hasPrefix("~") ? (trimmed as NSString).expandingTildeInPath : trimmed + let normalizedPattern = self.normalizeMatchTarget(expanded) + let normalizedTarget = self.normalizeMatchTarget(target) + guard let regex = self.regex(for: normalizedPattern) else { return false } + let range = NSRange(location: 0, length: normalizedTarget.utf16.count) + return regex.firstMatch(in: normalizedTarget, options: [], range: range) != nil + } + + private static func normalizeMatchTarget(_ value: String) -> String { + value.replacingOccurrences(of: "\\\\", with: "/").lowercased() + } + + private static func regex(for pattern: String) -> NSRegularExpression? { + var regex = "^" + var idx = pattern.startIndex + while idx < pattern.endIndex { + let ch = pattern[idx] + if ch == "*" { + let next = pattern.index(after: idx) + if next < pattern.endIndex, pattern[next] == "*" { + regex += ".*" + idx = pattern.index(after: next) + } else { + regex += "[^/]*" + idx = next + } + continue + } + if ch == "?" { + regex += "." + idx = pattern.index(after: idx) + continue + } + regex += NSRegularExpression.escapedPattern(for: String(ch)) + idx = pattern.index(after: idx) + } + regex += "$" + return try? NSRegularExpression(pattern: regex, options: [.caseInsensitive]) + } +} + +struct ExecEventPayload: Codable, Sendable { + var sessionKey: String + var runId: String + var host: String + var command: String? + var exitCode: Int? + var timedOut: Bool? + var success: Bool? + var output: String? + var reason: String? + + static func truncateOutput(_ raw: String, maxChars: Int = 20000) -> String? { + let trimmed = raw.trimmingCharacters(in: .whitespacesAndNewlines) + guard !trimmed.isEmpty else { return nil } + if trimmed.count <= maxChars { return trimmed } + let suffix = trimmed.suffix(maxChars) + return "... (truncated) \(suffix)" + } +} + +actor SkillBinsCache { + static let shared = SkillBinsCache() + + private var bins: Set = [] + private var lastRefresh: Date? + private let refreshInterval: TimeInterval = 90 + + func currentBins(force: Bool = false) async -> Set { + if force || self.isStale() { + await self.refresh() + } + return self.bins + } + + func refresh() async { + do { + let report = try await GatewayConnection.shared.skillsStatus() + var next = Set() + for skill in report.skills { + for bin in skill.requirements.bins { + let trimmed = bin.trimmingCharacters(in: .whitespacesAndNewlines) + if !trimmed.isEmpty { next.insert(trimmed) } + } + } + self.bins = next + self.lastRefresh = Date() + } catch { + if self.lastRefresh == nil { + self.bins = [] + } + } + } + + private func isStale() -> Bool { + guard let lastRefresh else { return true } + return Date().timeIntervalSince(lastRefresh) > self.refreshInterval + } +} diff --git a/apps/macos/Sources/OpenClaw/ExecApprovalsGatewayPrompter.swift b/apps/macos/Sources/OpenClaw/ExecApprovalsGatewayPrompter.swift new file mode 100644 index 0000000000000000000000000000000000000000..add04c73087ba4f09873b14fc8f5fd42dcd8404f --- /dev/null +++ b/apps/macos/Sources/OpenClaw/ExecApprovalsGatewayPrompter.swift @@ -0,0 +1,123 @@ +import OpenClawKit +import OpenClawProtocol +import CoreGraphics +import Foundation +import OSLog + +@MainActor +final class ExecApprovalsGatewayPrompter { + static let shared = ExecApprovalsGatewayPrompter() + + private let logger = Logger(subsystem: "ai.openclaw", category: "exec-approvals.gateway") + private var task: Task? + + struct GatewayApprovalRequest: Codable, Sendable { + var id: String + var request: ExecApprovalPromptRequest + var createdAtMs: Int + var expiresAtMs: Int + } + + func start() { + guard self.task == nil else { return } + self.task = Task { [weak self] in + await self?.run() + } + } + + func stop() { + self.task?.cancel() + self.task = nil + } + + private func run() async { + let stream = await GatewayConnection.shared.subscribe(bufferingNewest: 200) + for await push in stream { + if Task.isCancelled { return } + await self.handle(push: push) + } + } + + private func handle(push: GatewayPush) async { + guard case let .event(evt) = push else { return } + guard evt.event == "exec.approval.requested" else { return } + guard let payload = evt.payload else { return } + do { + let data = try JSONEncoder().encode(payload) + let request = try JSONDecoder().decode(GatewayApprovalRequest.self, from: data) + guard self.shouldPresent(request: request) else { return } + let decision = ExecApprovalsPromptPresenter.prompt(request.request) + try await GatewayConnection.shared.requestVoid( + method: .execApprovalResolve, + params: [ + "id": AnyCodable(request.id), + "decision": AnyCodable(decision.rawValue), + ], + timeoutMs: 10000) + } catch { + self.logger.error("exec approval handling failed \(error.localizedDescription, privacy: .public)") + } + } + + private func shouldPresent(request: GatewayApprovalRequest) -> Bool { + let mode = AppStateStore.shared.connectionMode + let activeSession = WebChatManager.shared.activeSessionKey?.trimmingCharacters(in: .whitespacesAndNewlines) + let requestSession = request.request.sessionKey?.trimmingCharacters(in: .whitespacesAndNewlines) + return Self.shouldPresent( + mode: mode, + activeSession: activeSession, + requestSession: requestSession, + lastInputSeconds: Self.lastInputSeconds(), + thresholdSeconds: 120) + } + + private static func shouldPresent( + mode: AppState.ConnectionMode, + activeSession: String?, + requestSession: String?, + lastInputSeconds: Int?, + thresholdSeconds: Int) -> Bool + { + let active = activeSession?.trimmingCharacters(in: .whitespacesAndNewlines) + let requested = requestSession?.trimmingCharacters(in: .whitespacesAndNewlines) + let recentlyActive = lastInputSeconds.map { $0 <= thresholdSeconds } ?? (mode == .local) + + if let session = requested, !session.isEmpty { + if let active, !active.isEmpty { + return active == session + } + return recentlyActive + } + + if let active, !active.isEmpty { + return true + } + return mode == .local + } + + private static func lastInputSeconds() -> Int? { + let anyEvent = CGEventType(rawValue: UInt32.max) ?? .null + let seconds = CGEventSource.secondsSinceLastEventType(.combinedSessionState, eventType: anyEvent) + if seconds.isNaN || seconds.isInfinite || seconds < 0 { return nil } + return Int(seconds.rounded()) + } +} + +#if DEBUG +extension ExecApprovalsGatewayPrompter { + static func _testShouldPresent( + mode: AppState.ConnectionMode, + activeSession: String?, + requestSession: String?, + lastInputSeconds: Int?, + thresholdSeconds: Int = 120) -> Bool + { + self.shouldPresent( + mode: mode, + activeSession: activeSession, + requestSession: requestSession, + lastInputSeconds: lastInputSeconds, + thresholdSeconds: thresholdSeconds) + } +} +#endif diff --git a/apps/macos/Sources/OpenClaw/ExecApprovalsSocket.swift b/apps/macos/Sources/OpenClaw/ExecApprovalsSocket.swift new file mode 100644 index 0000000000000000000000000000000000000000..f6d88c483026b937284fd41fb77d13956db26f09 --- /dev/null +++ b/apps/macos/Sources/OpenClaw/ExecApprovalsSocket.swift @@ -0,0 +1,831 @@ +import AppKit +import OpenClawKit +import CryptoKit +import Darwin +import Foundation +import OSLog + +struct ExecApprovalPromptRequest: Codable, Sendable { + var command: String + var cwd: String? + var host: String? + var security: String? + var ask: String? + var agentId: String? + var resolvedPath: String? + var sessionKey: String? +} + +private struct ExecApprovalSocketRequest: Codable { + var type: String + var token: String + var id: String + var request: ExecApprovalPromptRequest +} + +private struct ExecApprovalSocketDecision: Codable { + var type: String + var id: String + var decision: ExecApprovalDecision +} + +private struct ExecHostSocketRequest: Codable { + var type: String + var id: String + var nonce: String + var ts: Int + var hmac: String + var requestJson: String +} + +private struct ExecHostRequest: Codable { + var command: [String] + var rawCommand: String? + var cwd: String? + var env: [String: String]? + var timeoutMs: Int? + var needsScreenRecording: Bool? + var agentId: String? + var sessionKey: String? + var approvalDecision: ExecApprovalDecision? +} + +private struct ExecHostRunResult: Codable { + var exitCode: Int? + var timedOut: Bool + var success: Bool + var stdout: String + var stderr: String + var error: String? +} + +private struct ExecHostError: Codable { + var code: String + var message: String + var reason: String? +} + +private struct ExecHostResponse: Codable { + var type: String + var id: String + var ok: Bool + var payload: ExecHostRunResult? + var error: ExecHostError? +} + +enum ExecApprovalsSocketClient { + private struct TimeoutError: LocalizedError { + var message: String + var errorDescription: String? { self.message } + } + + static func requestDecision( + socketPath: String, + token: String, + request: ExecApprovalPromptRequest, + timeoutMs: Int = 15000) async -> ExecApprovalDecision? + { + let trimmedPath = socketPath.trimmingCharacters(in: .whitespacesAndNewlines) + let trimmedToken = token.trimmingCharacters(in: .whitespacesAndNewlines) + guard !trimmedPath.isEmpty, !trimmedToken.isEmpty else { return nil } + do { + return try await AsyncTimeout.withTimeoutMs( + timeoutMs: timeoutMs, + onTimeout: { + TimeoutError(message: "exec approvals socket timeout") + }, + operation: { + try await Task.detached { + try self.requestDecisionSync( + socketPath: trimmedPath, + token: trimmedToken, + request: request) + }.value + }) + } catch { + return nil + } + } + + private static func requestDecisionSync( + socketPath: String, + token: String, + request: ExecApprovalPromptRequest) throws -> ExecApprovalDecision? + { + let fd = socket(AF_UNIX, SOCK_STREAM, 0) + guard fd >= 0 else { + throw NSError(domain: "ExecApprovals", code: 1, userInfo: [ + NSLocalizedDescriptionKey: "socket create failed", + ]) + } + + var addr = sockaddr_un() + addr.sun_family = sa_family_t(AF_UNIX) + let maxLen = MemoryLayout.size(ofValue: addr.sun_path) + if socketPath.utf8.count >= maxLen { + throw NSError(domain: "ExecApprovals", code: 2, userInfo: [ + NSLocalizedDescriptionKey: "socket path too long", + ]) + } + socketPath.withCString { cstr in + withUnsafeMutablePointer(to: &addr.sun_path) { ptr in + let raw = UnsafeMutableRawPointer(ptr).assumingMemoryBound(to: Int8.self) + strncpy(raw, cstr, maxLen - 1) + } + } + let size = socklen_t(MemoryLayout.size(ofValue: addr)) + let result = withUnsafePointer(to: &addr) { ptr in + ptr.withMemoryRebound(to: sockaddr.self, capacity: 1) { rebound in + connect(fd, rebound, size) + } + } + if result != 0 { + throw NSError(domain: "ExecApprovals", code: 3, userInfo: [ + NSLocalizedDescriptionKey: "socket connect failed", + ]) + } + + let handle = FileHandle(fileDescriptor: fd, closeOnDealloc: true) + + let message = ExecApprovalSocketRequest( + type: "request", + token: token, + id: UUID().uuidString, + request: request) + let data = try JSONEncoder().encode(message) + var payload = data + payload.append(0x0A) + try handle.write(contentsOf: payload) + + guard let line = try self.readLine(from: handle, maxBytes: 256_000), + let lineData = line.data(using: .utf8) + else { return nil } + let response = try JSONDecoder().decode(ExecApprovalSocketDecision.self, from: lineData) + return response.decision + } + + private static func readLine(from handle: FileHandle, maxBytes: Int) throws -> String? { + var buffer = Data() + while buffer.count < maxBytes { + let chunk = try handle.read(upToCount: 4096) ?? Data() + if chunk.isEmpty { break } + buffer.append(chunk) + if buffer.contains(0x0A) { break } + } + guard let newlineIndex = buffer.firstIndex(of: 0x0A) else { + guard !buffer.isEmpty else { return nil } + return String(data: buffer, encoding: .utf8) + } + let lineData = buffer.subdata(in: 0.. ExecApprovalDecision { + NSApp.activate(ignoringOtherApps: true) + let alert = NSAlert() + alert.alertStyle = .warning + alert.messageText = "Allow this command?" + alert.informativeText = "Review the command details before allowing." + alert.accessoryView = self.buildAccessoryView(request) + + alert.addButton(withTitle: "Allow Once") + alert.addButton(withTitle: "Always Allow") + alert.addButton(withTitle: "Don't Allow") + if #available(macOS 11.0, *), alert.buttons.indices.contains(2) { + alert.buttons[2].hasDestructiveAction = true + } + + switch alert.runModal() { + case .alertFirstButtonReturn: + return .allowOnce + case .alertSecondButtonReturn: + return .allowAlways + default: + return .deny + } + } + + @MainActor + private static func buildAccessoryView(_ request: ExecApprovalPromptRequest) -> NSView { + let stack = NSStackView() + stack.orientation = .vertical + stack.spacing = 8 + stack.alignment = .leading + + let commandTitle = NSTextField(labelWithString: "Command") + commandTitle.font = NSFont.boldSystemFont(ofSize: NSFont.systemFontSize) + stack.addArrangedSubview(commandTitle) + + let commandText = NSTextView() + commandText.isEditable = false + commandText.isSelectable = true + commandText.drawsBackground = true + commandText.backgroundColor = NSColor.textBackgroundColor + commandText.font = NSFont.monospacedSystemFont(ofSize: NSFont.systemFontSize, weight: .regular) + commandText.string = request.command + commandText.textContainerInset = NSSize(width: 6, height: 6) + commandText.textContainer?.lineFragmentPadding = 0 + commandText.textContainer?.widthTracksTextView = true + commandText.isHorizontallyResizable = false + commandText.isVerticallyResizable = false + + let commandScroll = NSScrollView() + commandScroll.borderType = .lineBorder + commandScroll.hasVerticalScroller = false + commandScroll.hasHorizontalScroller = false + commandScroll.documentView = commandText + commandScroll.translatesAutoresizingMaskIntoConstraints = false + commandScroll.widthAnchor.constraint(lessThanOrEqualToConstant: 440).isActive = true + commandScroll.heightAnchor.constraint(greaterThanOrEqualToConstant: 56).isActive = true + stack.addArrangedSubview(commandScroll) + + let contextTitle = NSTextField(labelWithString: "Context") + contextTitle.font = NSFont.boldSystemFont(ofSize: NSFont.systemFontSize) + stack.addArrangedSubview(contextTitle) + + let contextStack = NSStackView() + contextStack.orientation = .vertical + contextStack.spacing = 4 + contextStack.alignment = .leading + + let trimmedCwd = request.cwd?.trimmingCharacters(in: .whitespacesAndNewlines) ?? "" + if !trimmedCwd.isEmpty { + self.addDetailRow(title: "Working directory", value: trimmedCwd, to: contextStack) + } + let trimmedAgent = request.agentId?.trimmingCharacters(in: .whitespacesAndNewlines) ?? "" + if !trimmedAgent.isEmpty { + self.addDetailRow(title: "Agent", value: trimmedAgent, to: contextStack) + } + let trimmedPath = request.resolvedPath?.trimmingCharacters(in: .whitespacesAndNewlines) ?? "" + if !trimmedPath.isEmpty { + self.addDetailRow(title: "Executable", value: trimmedPath, to: contextStack) + } + let trimmedHost = request.host?.trimmingCharacters(in: .whitespacesAndNewlines) ?? "" + if !trimmedHost.isEmpty { + self.addDetailRow(title: "Host", value: trimmedHost, to: contextStack) + } + if let security = request.security?.trimmingCharacters(in: .whitespacesAndNewlines), !security.isEmpty { + self.addDetailRow(title: "Security", value: security, to: contextStack) + } + if let ask = request.ask?.trimmingCharacters(in: .whitespacesAndNewlines), !ask.isEmpty { + self.addDetailRow(title: "Ask mode", value: ask, to: contextStack) + } + + if contextStack.arrangedSubviews.isEmpty { + let empty = NSTextField(labelWithString: "No additional context provided.") + empty.textColor = NSColor.secondaryLabelColor + empty.font = NSFont.systemFont(ofSize: NSFont.smallSystemFontSize) + contextStack.addArrangedSubview(empty) + } + + stack.addArrangedSubview(contextStack) + + let footer = NSTextField(labelWithString: "This runs on this machine.") + footer.textColor = NSColor.secondaryLabelColor + footer.font = NSFont.systemFont(ofSize: NSFont.smallSystemFontSize) + stack.addArrangedSubview(footer) + + return stack + } + + @MainActor + private static func addDetailRow(title: String, value: String, to stack: NSStackView) { + let row = NSStackView() + row.orientation = .horizontal + row.spacing = 6 + row.alignment = .firstBaseline + + let titleLabel = NSTextField(labelWithString: "\(title):") + titleLabel.font = NSFont.systemFont(ofSize: NSFont.smallSystemFontSize, weight: .semibold) + titleLabel.textColor = NSColor.secondaryLabelColor + + let valueLabel = NSTextField(labelWithString: value) + valueLabel.font = NSFont.systemFont(ofSize: NSFont.smallSystemFontSize) + valueLabel.lineBreakMode = .byTruncatingMiddle + valueLabel.setContentCompressionResistancePriority(.defaultLow, for: .horizontal) + + row.addArrangedSubview(titleLabel) + row.addArrangedSubview(valueLabel) + stack.addArrangedSubview(row) + } +} + +@MainActor +private enum ExecHostExecutor { + private struct ExecApprovalContext { + let command: [String] + let displayCommand: String + let trimmedAgent: String? + let approvals: ExecApprovalsResolved + let security: ExecSecurity + let ask: ExecAsk + let autoAllowSkills: Bool + let env: [String: String]? + let resolution: ExecCommandResolution? + let allowlistMatch: ExecAllowlistEntry? + let skillAllow: Bool + } + + private static let blockedEnvKeys: Set = [ + "PATH", + "NODE_OPTIONS", + "PYTHONHOME", + "PYTHONPATH", + "PERL5LIB", + "PERL5OPT", + "RUBYOPT", + ] + + private static let blockedEnvPrefixes: [String] = [ + "DYLD_", + "LD_", + ] + + static func handle(_ request: ExecHostRequest) async -> ExecHostResponse { + let command = request.command.map { $0.trimmingCharacters(in: .whitespacesAndNewlines) } + guard !command.isEmpty else { + return self.errorResponse( + code: "INVALID_REQUEST", + message: "command required", + reason: "invalid") + } + + let context = await self.buildContext(request: request, command: command) + if context.security == .deny { + return self.errorResponse( + code: "UNAVAILABLE", + message: "SYSTEM_RUN_DISABLED: security=deny", + reason: "security=deny") + } + + let approvalDecision = request.approvalDecision + if approvalDecision == .deny { + return self.errorResponse( + code: "UNAVAILABLE", + message: "SYSTEM_RUN_DENIED: user denied", + reason: "user-denied") + } + + var approvedByAsk = approvalDecision != nil + if ExecApprovalHelpers.requiresAsk( + ask: context.ask, + security: context.security, + allowlistMatch: context.allowlistMatch, + skillAllow: context.skillAllow), + approvalDecision == nil + { + let decision = ExecApprovalsPromptPresenter.prompt( + ExecApprovalPromptRequest( + command: context.displayCommand, + cwd: request.cwd, + host: "node", + security: context.security.rawValue, + ask: context.ask.rawValue, + agentId: context.trimmedAgent, + resolvedPath: context.resolution?.resolvedPath, + sessionKey: request.sessionKey)) + + switch decision { + case .deny: + return self.errorResponse( + code: "UNAVAILABLE", + message: "SYSTEM_RUN_DENIED: user denied", + reason: "user-denied") + case .allowAlways: + approvedByAsk = true + self.persistAllowlistEntry(decision: decision, context: context) + case .allowOnce: + approvedByAsk = true + } + } + + self.persistAllowlistEntry(decision: approvalDecision, context: context) + + if context.security == .allowlist, + context.allowlistMatch == nil, + !context.skillAllow, + !approvedByAsk + { + return self.errorResponse( + code: "UNAVAILABLE", + message: "SYSTEM_RUN_DENIED: allowlist miss", + reason: "allowlist-miss") + } + + if let match = context.allowlistMatch { + ExecApprovalsStore.recordAllowlistUse( + agentId: context.trimmedAgent, + pattern: match.pattern, + command: context.displayCommand, + resolvedPath: context.resolution?.resolvedPath) + } + + if let errorResponse = await self.ensureScreenRecordingAccess(request.needsScreenRecording) { + return errorResponse + } + + return await self.runCommand( + command: command, + cwd: request.cwd, + env: context.env, + timeoutMs: request.timeoutMs) + } + + private static func buildContext(request: ExecHostRequest, command: [String]) async -> ExecApprovalContext { + let displayCommand = ExecCommandFormatter.displayString( + for: command, + rawCommand: request.rawCommand) + let agentId = request.agentId?.trimmingCharacters(in: .whitespacesAndNewlines) + let trimmedAgent = (agentId?.isEmpty == false) ? agentId : nil + let approvals = ExecApprovalsStore.resolve(agentId: trimmedAgent) + let security = approvals.agent.security + let ask = approvals.agent.ask + let autoAllowSkills = approvals.agent.autoAllowSkills + let env = self.sanitizedEnv(request.env) + let resolution = ExecCommandResolution.resolve( + command: command, + rawCommand: request.rawCommand, + cwd: request.cwd, + env: env) + let allowlistMatch = security == .allowlist + ? ExecAllowlistMatcher.match(entries: approvals.allowlist, resolution: resolution) + : nil + let skillAllow: Bool + if autoAllowSkills, let name = resolution?.executableName { + let bins = await SkillBinsCache.shared.currentBins() + skillAllow = bins.contains(name) + } else { + skillAllow = false + } + return ExecApprovalContext( + command: command, + displayCommand: displayCommand, + trimmedAgent: trimmedAgent, + approvals: approvals, + security: security, + ask: ask, + autoAllowSkills: autoAllowSkills, + env: env, + resolution: resolution, + allowlistMatch: allowlistMatch, + skillAllow: skillAllow) + } + + private static func persistAllowlistEntry( + decision: ExecApprovalDecision?, + context: ExecApprovalContext) + { + guard decision == .allowAlways, context.security == .allowlist else { return } + guard let pattern = ExecApprovalHelpers.allowlistPattern( + command: context.command, + resolution: context.resolution) + else { + return + } + ExecApprovalsStore.addAllowlistEntry(agentId: context.trimmedAgent, pattern: pattern) + } + + private static func ensureScreenRecordingAccess(_ needsScreenRecording: Bool?) async -> ExecHostResponse? { + guard needsScreenRecording == true else { return nil } + let authorized = await PermissionManager + .status([.screenRecording])[.screenRecording] ?? false + if authorized { return nil } + return self.errorResponse( + code: "UNAVAILABLE", + message: "PERMISSION_MISSING: screenRecording", + reason: "permission:screenRecording") + } + + private static func runCommand( + command: [String], + cwd: String?, + env: [String: String]?, + timeoutMs: Int?) async -> ExecHostResponse + { + let timeoutSec = timeoutMs.flatMap { Double($0) / 1000.0 } + let result = await Task.detached { () -> ShellExecutor.ShellResult in + await ShellExecutor.runDetailed( + command: command, + cwd: cwd, + env: env, + timeout: timeoutSec) + }.value + let payload = ExecHostRunResult( + exitCode: result.exitCode, + timedOut: result.timedOut, + success: result.success, + stdout: result.stdout, + stderr: result.stderr, + error: result.errorMessage) + return self.successResponse(payload) + } + + private static func errorResponse( + code: String, + message: String, + reason: String?) -> ExecHostResponse + { + ExecHostResponse( + type: "exec-res", + id: UUID().uuidString, + ok: false, + payload: nil, + error: ExecHostError(code: code, message: message, reason: reason)) + } + + private static func successResponse(_ payload: ExecHostRunResult) -> ExecHostResponse { + ExecHostResponse( + type: "exec-res", + id: UUID().uuidString, + ok: true, + payload: payload, + error: nil) + } + + private static func sanitizedEnv(_ overrides: [String: String]?) -> [String: String]? { + guard let overrides else { return nil } + var merged = ProcessInfo.processInfo.environment + for (rawKey, value) in overrides { + let key = rawKey.trimmingCharacters(in: .whitespacesAndNewlines) + guard !key.isEmpty else { continue } + let upper = key.uppercased() + if self.blockedEnvKeys.contains(upper) { continue } + if self.blockedEnvPrefixes.contains(where: { upper.hasPrefix($0) }) { continue } + merged[key] = value + } + return merged + } +} + +private final class ExecApprovalsSocketServer: @unchecked Sendable { + private let logger = Logger(subsystem: "ai.openclaw", category: "exec-approvals.socket") + private let socketPath: String + private let token: String + private let onPrompt: @Sendable (ExecApprovalPromptRequest) async -> ExecApprovalDecision + private let onExec: @Sendable (ExecHostRequest) async -> ExecHostResponse + private var socketFD: Int32 = -1 + private var acceptTask: Task? + private var isRunning = false + + init( + socketPath: String, + token: String, + onPrompt: @escaping @Sendable (ExecApprovalPromptRequest) async -> ExecApprovalDecision, + onExec: @escaping @Sendable (ExecHostRequest) async -> ExecHostResponse) + { + self.socketPath = socketPath + self.token = token + self.onPrompt = onPrompt + self.onExec = onExec + } + + func start() { + guard !self.isRunning else { return } + self.isRunning = true + self.acceptTask = Task.detached { [weak self] in + await self?.runAcceptLoop() + } + } + + func stop() { + self.isRunning = false + self.acceptTask?.cancel() + self.acceptTask = nil + if self.socketFD >= 0 { + close(self.socketFD) + self.socketFD = -1 + } + if !self.socketPath.isEmpty { + unlink(self.socketPath) + } + } + + private func runAcceptLoop() async { + let fd = self.openSocket() + guard fd >= 0 else { + self.isRunning = false + return + } + self.socketFD = fd + while self.isRunning { + var addr = sockaddr_un() + var len = socklen_t(MemoryLayout.size(ofValue: addr)) + let client = withUnsafeMutablePointer(to: &addr) { ptr in + ptr.withMemoryRebound(to: sockaddr.self, capacity: 1) { rebound in + accept(fd, rebound, &len) + } + } + if client < 0 { + if errno == EINTR { continue } + break + } + Task.detached { [weak self] in + await self?.handleClient(fd: client) + } + } + } + + private func openSocket() -> Int32 { + let fd = socket(AF_UNIX, SOCK_STREAM, 0) + guard fd >= 0 else { + self.logger.error("exec approvals socket create failed") + return -1 + } + unlink(self.socketPath) + var addr = sockaddr_un() + addr.sun_family = sa_family_t(AF_UNIX) + let maxLen = MemoryLayout.size(ofValue: addr.sun_path) + if self.socketPath.utf8.count >= maxLen { + self.logger.error("exec approvals socket path too long") + close(fd) + return -1 + } + self.socketPath.withCString { cstr in + withUnsafeMutablePointer(to: &addr.sun_path) { ptr in + let raw = UnsafeMutableRawPointer(ptr).assumingMemoryBound(to: Int8.self) + memset(raw, 0, maxLen) + strncpy(raw, cstr, maxLen - 1) + } + } + let size = socklen_t(MemoryLayout.size(ofValue: addr)) + let result = withUnsafePointer(to: &addr) { ptr in + ptr.withMemoryRebound(to: sockaddr.self, capacity: 1) { rebound in + bind(fd, rebound, size) + } + } + if result != 0 { + self.logger.error("exec approvals socket bind failed") + close(fd) + return -1 + } + if listen(fd, 16) != 0 { + self.logger.error("exec approvals socket listen failed") + close(fd) + return -1 + } + chmod(self.socketPath, 0o600) + self.logger.info("exec approvals socket listening at \(self.socketPath, privacy: .public)") + return fd + } + + private func handleClient(fd: Int32) async { + let handle = FileHandle(fileDescriptor: fd, closeOnDealloc: true) + do { + guard self.isAllowedPeer(fd: fd) else { + try self.sendApprovalResponse(handle: handle, id: UUID().uuidString, decision: .deny) + return + } + guard let line = try self.readLine(from: handle, maxBytes: 256_000), + let data = line.data(using: .utf8) + else { + return + } + guard + let envelope = try JSONSerialization.jsonObject(with: data) as? [String: Any], + let type = envelope["type"] as? String + else { + return + } + + if type == "request" { + let request = try JSONDecoder().decode(ExecApprovalSocketRequest.self, from: data) + guard request.token == self.token else { + try self.sendApprovalResponse(handle: handle, id: request.id, decision: .deny) + return + } + let decision = await self.onPrompt(request.request) + try self.sendApprovalResponse(handle: handle, id: request.id, decision: decision) + return + } + + if type == "exec" { + let request = try JSONDecoder().decode(ExecHostSocketRequest.self, from: data) + let response = await self.handleExecRequest(request) + try self.sendExecResponse(handle: handle, response: response) + return + } + } catch { + self.logger.error("exec approvals socket handling failed: \(error.localizedDescription, privacy: .public)") + } + } + + private func readLine(from handle: FileHandle, maxBytes: Int) throws -> String? { + var buffer = Data() + while buffer.count < maxBytes { + let chunk = try handle.read(upToCount: 4096) ?? Data() + if chunk.isEmpty { break } + buffer.append(chunk) + if buffer.contains(0x0A) { break } + } + guard let newlineIndex = buffer.firstIndex(of: 0x0A) else { + guard !buffer.isEmpty else { return nil } + return String(data: buffer, encoding: .utf8) + } + let lineData = buffer.subdata(in: 0.. Bool { + var uid = uid_t(0) + var gid = gid_t(0) + if getpeereid(fd, &uid, &gid) != 0 { + return false + } + return uid == geteuid() + } + + private func handleExecRequest(_ request: ExecHostSocketRequest) async -> ExecHostResponse { + let nowMs = Int(Date().timeIntervalSince1970 * 1000) + if abs(nowMs - request.ts) > 10000 { + return ExecHostResponse( + type: "exec-res", + id: request.id, + ok: false, + payload: nil, + error: ExecHostError(code: "INVALID_REQUEST", message: "expired request", reason: "ttl")) + } + let expected = self.hmacHex(nonce: request.nonce, ts: request.ts, requestJson: request.requestJson) + if expected != request.hmac { + return ExecHostResponse( + type: "exec-res", + id: request.id, + ok: false, + payload: nil, + error: ExecHostError(code: "INVALID_REQUEST", message: "invalid auth", reason: "hmac")) + } + guard let requestData = request.requestJson.data(using: .utf8), + let payload = try? JSONDecoder().decode(ExecHostRequest.self, from: requestData) + else { + return ExecHostResponse( + type: "exec-res", + id: request.id, + ok: false, + payload: nil, + error: ExecHostError(code: "INVALID_REQUEST", message: "invalid payload", reason: "json")) + } + let response = await self.onExec(payload) + return ExecHostResponse( + type: "exec-res", + id: request.id, + ok: response.ok, + payload: response.payload, + error: response.error) + } + + private func hmacHex(nonce: String, ts: Int, requestJson: String) -> String { + let key = SymmetricKey(data: Data(self.token.utf8)) + let message = "\(nonce):\(ts):\(requestJson)" + let mac = HMAC.authenticationCode(for: Data(message.utf8), using: key) + return mac.map { String(format: "%02x", $0) }.joined() + } +} diff --git a/apps/macos/Sources/OpenClaw/FileHandle+SafeRead.swift b/apps/macos/Sources/OpenClaw/FileHandle+SafeRead.swift new file mode 100644 index 0000000000000000000000000000000000000000..7cd160969389439255d76b608b0ec3a6cb1f1850 --- /dev/null +++ b/apps/macos/Sources/OpenClaw/FileHandle+SafeRead.swift @@ -0,0 +1,28 @@ +import Foundation + +extension FileHandle { + /// Reads until EOF using the throwing FileHandle API and returns empty `Data` on failure. + /// + /// Important: Avoid legacy, non-throwing FileHandle read APIs (e.g. `readDataToEndOfFile()` and + /// `availableData`). They can raise Objective-C exceptions when the handle is closed/invalid, which + /// will abort the process. + func readToEndSafely() -> Data { + do { + return try self.readToEnd() ?? Data() + } catch { + return Data() + } + } + + /// Reads up to `count` bytes using the throwing FileHandle API and returns empty `Data` on failure/EOF. + /// + /// Important: Use this instead of `availableData` in callbacks like `readabilityHandler` to avoid + /// Objective-C exceptions terminating the process. + func readSafely(upToCount count: Int) -> Data { + do { + return try self.read(upToCount: count) ?? Data() + } catch { + return Data() + } + } +} diff --git a/apps/macos/Sources/OpenClaw/GatewayAutostartPolicy.swift b/apps/macos/Sources/OpenClaw/GatewayAutostartPolicy.swift new file mode 100644 index 0000000000000000000000000000000000000000..27f60abadb640b1d6f5d8a7560f9a1a723ea43b5 --- /dev/null +++ b/apps/macos/Sources/OpenClaw/GatewayAutostartPolicy.swift @@ -0,0 +1,14 @@ +import Foundation + +enum GatewayAutostartPolicy { + static func shouldStartGateway(mode: AppState.ConnectionMode, paused: Bool) -> Bool { + mode == .local && !paused + } + + static func shouldEnsureLaunchAgent( + mode: AppState.ConnectionMode, + paused: Bool) -> Bool + { + self.shouldStartGateway(mode: mode, paused: paused) + } +} diff --git a/apps/macos/Sources/OpenClaw/GatewayConnection.swift b/apps/macos/Sources/OpenClaw/GatewayConnection.swift new file mode 100644 index 0000000000000000000000000000000000000000..f7509236dcc71c11ce5852b0a75e0a9aaa2f1ae1 --- /dev/null +++ b/apps/macos/Sources/OpenClaw/GatewayConnection.swift @@ -0,0 +1,737 @@ +import OpenClawChatUI +import OpenClawKit +import OpenClawProtocol +import Foundation +import OSLog + +private let gatewayConnectionLogger = Logger(subsystem: "ai.openclaw", category: "gateway.connection") + +enum GatewayAgentChannel: String, Codable, CaseIterable, Sendable { + case last + case whatsapp + case telegram + case discord + case googlechat + case slack + case signal + case imessage + case msteams + case bluebubbles + case webchat + + init(raw: String?) { + let normalized = (raw ?? "").trimmingCharacters(in: .whitespacesAndNewlines).lowercased() + self = GatewayAgentChannel(rawValue: normalized) ?? .last + } + + var isDeliverable: Bool { self != .webchat } + + func shouldDeliver(_ deliver: Bool) -> Bool { deliver && self.isDeliverable } +} + +struct GatewayAgentInvocation: Sendable { + var message: String + var sessionKey: String = "main" + var thinking: String? + var deliver: Bool = false + var to: String? + var channel: GatewayAgentChannel = .last + var timeoutSeconds: Int? + var idempotencyKey: String = UUID().uuidString +} + +/// Single, shared Gateway websocket connection for the whole app. +/// +/// This owns exactly one `GatewayChannelActor` and reuses it across all callers +/// (ControlChannel, debug actions, SwiftUI WebChat, etc.). +actor GatewayConnection { + static let shared = GatewayConnection() + + typealias Config = (url: URL, token: String?, password: String?) + + enum Method: String, Sendable { + case agent + case status + case setHeartbeats = "set-heartbeats" + case systemEvent = "system-event" + case health + case channelsStatus = "channels.status" + case configGet = "config.get" + case configSet = "config.set" + case configPatch = "config.patch" + case configSchema = "config.schema" + case wizardStart = "wizard.start" + case wizardNext = "wizard.next" + case wizardCancel = "wizard.cancel" + case wizardStatus = "wizard.status" + case talkMode = "talk.mode" + case webLoginStart = "web.login.start" + case webLoginWait = "web.login.wait" + case channelsLogout = "channels.logout" + case modelsList = "models.list" + case chatHistory = "chat.history" + case sessionsPreview = "sessions.preview" + case chatSend = "chat.send" + case chatAbort = "chat.abort" + case skillsStatus = "skills.status" + case skillsInstall = "skills.install" + case skillsUpdate = "skills.update" + case voicewakeGet = "voicewake.get" + case voicewakeSet = "voicewake.set" + case nodePairApprove = "node.pair.approve" + case nodePairReject = "node.pair.reject" + case devicePairList = "device.pair.list" + case devicePairApprove = "device.pair.approve" + case devicePairReject = "device.pair.reject" + case execApprovalResolve = "exec.approval.resolve" + case cronList = "cron.list" + case cronRuns = "cron.runs" + case cronRun = "cron.run" + case cronRemove = "cron.remove" + case cronUpdate = "cron.update" + case cronAdd = "cron.add" + case cronStatus = "cron.status" + } + + private let configProvider: @Sendable () async throws -> Config + private let sessionBox: WebSocketSessionBox? + private let decoder = JSONDecoder() + + private var client: GatewayChannelActor? + private var configuredURL: URL? + private var configuredToken: String? + private var configuredPassword: String? + + private var subscribers: [UUID: AsyncStream.Continuation] = [:] + private var lastSnapshot: HelloOk? + + init( + configProvider: @escaping @Sendable () async throws -> Config = GatewayConnection.defaultConfigProvider, + sessionBox: WebSocketSessionBox? = nil) + { + self.configProvider = configProvider + self.sessionBox = sessionBox + } + + // MARK: - Low-level request + + func request( + method: String, + params: [String: AnyCodable]?, + timeoutMs: Double? = nil) async throws -> Data + { + let cfg = try await self.configProvider() + await self.configure(url: cfg.url, token: cfg.token, password: cfg.password) + guard let client else { + throw NSError(domain: "Gateway", code: 0, userInfo: [NSLocalizedDescriptionKey: "gateway not configured"]) + } + + do { + return try await client.request(method: method, params: params, timeoutMs: timeoutMs) + } catch { + if error is GatewayResponseError || error is GatewayDecodingError { + throw error + } + + // Auto-recover in local mode by spawning/attaching a gateway and retrying a few times. + // Canvas interactions should "just work" even if the local gateway isn't running yet. + let mode = await MainActor.run { AppStateStore.shared.connectionMode } + switch mode { + case .local: + await MainActor.run { GatewayProcessManager.shared.setActive(true) } + + var lastError: Error = error + for delayMs in [150, 400, 900] { + try await Task.sleep(nanoseconds: UInt64(delayMs) * 1_000_000) + do { + return try await client.request(method: method, params: params, timeoutMs: timeoutMs) + } catch { + lastError = error + } + } + + let nsError = lastError as NSError + if nsError.domain == URLError.errorDomain, + let fallback = await GatewayEndpointStore.shared.maybeFallbackToTailnet(from: cfg.url) + { + await self.configure(url: fallback.url, token: fallback.token, password: fallback.password) + for delayMs in [150, 400, 900] { + try await Task.sleep(nanoseconds: UInt64(delayMs) * 1_000_000) + do { + guard let client = self.client else { + throw NSError( + domain: "Gateway", + code: 0, + userInfo: [NSLocalizedDescriptionKey: "gateway not configured"]) + } + return try await client.request(method: method, params: params, timeoutMs: timeoutMs) + } catch { + lastError = error + } + } + } + + throw lastError + case .remote: + let nsError = error as NSError + guard nsError.domain == URLError.errorDomain else { throw error } + + var lastError: Error = error + await RemoteTunnelManager.shared.stopAll() + do { + _ = try await GatewayEndpointStore.shared.ensureRemoteControlTunnel() + } catch { + lastError = error + } + + for delayMs in [150, 400, 900] { + try await Task.sleep(nanoseconds: UInt64(delayMs) * 1_000_000) + do { + let cfg = try await self.configProvider() + await self.configure(url: cfg.url, token: cfg.token, password: cfg.password) + guard let client = self.client else { + throw NSError( + domain: "Gateway", + code: 0, + userInfo: [NSLocalizedDescriptionKey: "gateway not configured"]) + } + return try await client.request(method: method, params: params, timeoutMs: timeoutMs) + } catch { + lastError = error + } + } + + throw lastError + case .unconfigured: + throw error + } + } + } + + func requestRaw( + method: Method, + params: [String: AnyCodable]? = nil, + timeoutMs: Double? = nil) async throws -> Data + { + try await self.request(method: method.rawValue, params: params, timeoutMs: timeoutMs) + } + + func requestRaw( + method: String, + params: [String: AnyCodable]? = nil, + timeoutMs: Double? = nil) async throws -> Data + { + try await self.request(method: method, params: params, timeoutMs: timeoutMs) + } + + func requestDecoded( + method: Method, + params: [String: AnyCodable]? = nil, + timeoutMs: Double? = nil) async throws -> T + { + let data = try await self.requestRaw(method: method, params: params, timeoutMs: timeoutMs) + do { + return try self.decoder.decode(T.self, from: data) + } catch { + throw GatewayDecodingError(method: method.rawValue, message: error.localizedDescription) + } + } + + func requestVoid( + method: Method, + params: [String: AnyCodable]? = nil, + timeoutMs: Double? = nil) async throws + { + _ = try await self.requestRaw(method: method, params: params, timeoutMs: timeoutMs) + } + + /// Ensure the underlying socket is configured (and replaced if config changed). + func refresh() async throws { + let cfg = try await self.configProvider() + await self.configure(url: cfg.url, token: cfg.token, password: cfg.password) + } + + func authSource() async -> GatewayAuthSource? { + guard let client else { return nil } + return await client.authSource() + } + + func shutdown() async { + if let client { + await client.shutdown() + } + self.client = nil + self.configuredURL = nil + self.configuredToken = nil + self.lastSnapshot = nil + } + + func canvasHostUrl() async -> String? { + guard let snapshot = self.lastSnapshot else { return nil } + let trimmed = snapshot.canvashosturl?.trimmingCharacters(in: CharacterSet.whitespacesAndNewlines) ?? "" + return trimmed.isEmpty ? nil : trimmed + } + + private func sessionDefaultString(_ defaults: [String: OpenClawProtocol.AnyCodable]?, key: String) -> String { + let raw = defaults?[key]?.value as? String + return (raw ?? "").trimmingCharacters(in: CharacterSet.whitespacesAndNewlines) + } + + func cachedMainSessionKey() -> String? { + guard let snapshot = self.lastSnapshot else { return nil } + let trimmed = self.sessionDefaultString(snapshot.snapshot.sessiondefaults, key: "mainSessionKey") + return trimmed.isEmpty ? nil : trimmed + } + + func cachedGatewayVersion() -> String? { + guard let snapshot = self.lastSnapshot else { return nil } + let raw = snapshot.server["version"]?.value as? String + let trimmed = raw?.trimmingCharacters(in: CharacterSet.whitespacesAndNewlines) ?? "" + return trimmed.isEmpty ? nil : trimmed + } + + func snapshotPaths() -> (configPath: String?, stateDir: String?) { + guard let snapshot = self.lastSnapshot else { return (nil, nil) } + let configPath = snapshot.snapshot.configpath?.trimmingCharacters(in: .whitespacesAndNewlines) + let stateDir = snapshot.snapshot.statedir?.trimmingCharacters(in: .whitespacesAndNewlines) + return ( + configPath?.isEmpty == false ? configPath : nil, + stateDir?.isEmpty == false ? stateDir : nil) + } + + func subscribe(bufferingNewest: Int = 100) -> AsyncStream { + let id = UUID() + let snapshot = self.lastSnapshot + let connection = self + return AsyncStream(bufferingPolicy: .bufferingNewest(bufferingNewest)) { continuation in + if let snapshot { + continuation.yield(.snapshot(snapshot)) + } + self.subscribers[id] = continuation + continuation.onTermination = { @Sendable _ in + Task { await connection.removeSubscriber(id) } + } + } + } + + private func removeSubscriber(_ id: UUID) { + self.subscribers[id] = nil + } + + private func broadcast(_ push: GatewayPush) { + if case let .snapshot(snapshot) = push { + self.lastSnapshot = snapshot + if let mainSessionKey = self.cachedMainSessionKey() { + Task { @MainActor in + WorkActivityStore.shared.setMainSessionKey(mainSessionKey) + } + } + } + for (_, continuation) in self.subscribers { + continuation.yield(push) + } + } + + private func canonicalizeSessionKey(_ raw: String) -> String { + let trimmed = raw.trimmingCharacters(in: CharacterSet.whitespacesAndNewlines) + guard !trimmed.isEmpty else { return trimmed } + guard let defaults = self.lastSnapshot?.snapshot.sessiondefaults else { return trimmed } + let mainSessionKey = self.sessionDefaultString(defaults, key: "mainSessionKey") + guard !mainSessionKey.isEmpty else { return trimmed } + let mainKey = self.sessionDefaultString(defaults, key: "mainKey") + let defaultAgentId = self.sessionDefaultString(defaults, key: "defaultAgentId") + let isMainAlias = + trimmed == "main" || + (!mainKey.isEmpty && trimmed == mainKey) || + trimmed == mainSessionKey || + (!defaultAgentId.isEmpty && + (trimmed == "agent:\(defaultAgentId):main" || + (mainKey.isEmpty == false && trimmed == "agent:\(defaultAgentId):\(mainKey)"))) + return isMainAlias ? mainSessionKey : trimmed + } + + private func configure(url: URL, token: String?, password: String?) async { + if self.client != nil, self.configuredURL == url, self.configuredToken == token, + self.configuredPassword == password + { + return + } + if let client { + await client.shutdown() + } + self.lastSnapshot = nil + self.client = GatewayChannelActor( + url: url, + token: token, + password: password, + session: self.sessionBox, + pushHandler: { [weak self] push in + await self?.handle(push: push) + }) + self.configuredURL = url + self.configuredToken = token + self.configuredPassword = password + } + + private func handle(push: GatewayPush) { + self.broadcast(push) + } + + private static func defaultConfigProvider() async throws -> Config { + try await GatewayEndpointStore.shared.requireConfig() + } +} + +// MARK: - Typed gateway API + +extension GatewayConnection { + struct ConfigGetSnapshot: Decodable, Sendable { + struct SnapshotConfig: Decodable, Sendable { + struct Session: Decodable, Sendable { + let mainKey: String? + let scope: String? + } + + let session: Session? + } + + let config: SnapshotConfig? + } + + static func mainSessionKey(fromConfigGetData data: Data) throws -> String { + let snapshot = try JSONDecoder().decode(ConfigGetSnapshot.self, from: data) + let scope = snapshot.config?.session?.scope?.trimmingCharacters(in: .whitespacesAndNewlines) + if scope == "global" { + return "global" + } + return "main" + } + + func mainSessionKey(timeoutMs: Double = 15000) async -> String { + if let cached = self.cachedMainSessionKey() { + return cached + } + do { + let data = try await self.requestRaw(method: "config.get", params: nil, timeoutMs: timeoutMs) + return try Self.mainSessionKey(fromConfigGetData: data) + } catch { + return "main" + } + } + + func status() async -> (ok: Bool, error: String?) { + do { + _ = try await self.requestRaw(method: .status) + return (true, nil) + } catch { + return (false, error.localizedDescription) + } + } + + func setHeartbeatsEnabled(_ enabled: Bool) async -> Bool { + do { + try await self.requestVoid(method: .setHeartbeats, params: ["enabled": AnyCodable(enabled)]) + return true + } catch { + gatewayConnectionLogger.error("setHeartbeatsEnabled failed \(error.localizedDescription, privacy: .public)") + return false + } + } + + func sendAgent(_ invocation: GatewayAgentInvocation) async -> (ok: Bool, error: String?) { + let trimmed = invocation.message.trimmingCharacters(in: .whitespacesAndNewlines) + guard !trimmed.isEmpty else { return (false, "message empty") } + let sessionKey = self.canonicalizeSessionKey(invocation.sessionKey) + + var params: [String: AnyCodable] = [ + "message": AnyCodable(trimmed), + "sessionKey": AnyCodable(sessionKey), + "thinking": AnyCodable(invocation.thinking ?? "default"), + "deliver": AnyCodable(invocation.deliver), + "to": AnyCodable(invocation.to ?? ""), + "channel": AnyCodable(invocation.channel.rawValue), + "idempotencyKey": AnyCodable(invocation.idempotencyKey), + ] + if let timeout = invocation.timeoutSeconds { + params["timeout"] = AnyCodable(timeout) + } + + do { + try await self.requestVoid(method: .agent, params: params) + return (true, nil) + } catch { + return (false, error.localizedDescription) + } + } + + func sendAgent( + message: String, + thinking: String?, + sessionKey: String, + deliver: Bool, + to: String?, + channel: GatewayAgentChannel = .last, + timeoutSeconds: Int? = nil, + idempotencyKey: String = UUID().uuidString) async -> (ok: Bool, error: String?) + { + await self.sendAgent(GatewayAgentInvocation( + message: message, + sessionKey: sessionKey, + thinking: thinking, + deliver: deliver, + to: to, + channel: channel, + timeoutSeconds: timeoutSeconds, + idempotencyKey: idempotencyKey)) + } + + func sendSystemEvent(_ params: [String: AnyCodable]) async { + do { + try await self.requestVoid(method: .systemEvent, params: params) + } catch { + // Best-effort only. + } + } + + // MARK: - Health + + func healthSnapshot(timeoutMs: Double? = nil) async throws -> HealthSnapshot { + let data = try await self.requestRaw(method: .health, timeoutMs: timeoutMs) + if let snap = decodeHealthSnapshot(from: data) { return snap } + throw GatewayDecodingError(method: Method.health.rawValue, message: "failed to decode health snapshot") + } + + func healthOK(timeoutMs: Int = 8000) async throws -> Bool { + let data = try await self.requestRaw(method: .health, timeoutMs: Double(timeoutMs)) + return (try? self.decoder.decode(OpenClawGatewayHealthOK.self, from: data))?.ok ?? true + } + + // MARK: - Skills + + func skillsStatus() async throws -> SkillsStatusReport { + try await self.requestDecoded(method: .skillsStatus) + } + + func skillsInstall( + name: String, + installId: String, + timeoutMs: Int? = nil) async throws -> SkillInstallResult + { + var params: [String: AnyCodable] = [ + "name": AnyCodable(name), + "installId": AnyCodable(installId), + ] + if let timeoutMs { + params["timeoutMs"] = AnyCodable(timeoutMs) + } + return try await self.requestDecoded(method: .skillsInstall, params: params) + } + + func skillsUpdate( + skillKey: String, + enabled: Bool? = nil, + apiKey: String? = nil, + env: [String: String]? = nil) async throws -> SkillUpdateResult + { + var params: [String: AnyCodable] = [ + "skillKey": AnyCodable(skillKey), + ] + if let enabled { params["enabled"] = AnyCodable(enabled) } + if let apiKey { params["apiKey"] = AnyCodable(apiKey) } + if let env, !env.isEmpty { params["env"] = AnyCodable(env) } + return try await self.requestDecoded(method: .skillsUpdate, params: params) + } + + // MARK: - Sessions + + func sessionsPreview( + keys: [String], + limit: Int? = nil, + maxChars: Int? = nil, + timeoutMs: Int? = nil) async throws -> OpenClawSessionsPreviewPayload + { + let resolvedKeys = keys + .map { self.canonicalizeSessionKey($0) } + .filter { !$0.isEmpty } + if resolvedKeys.isEmpty { + return OpenClawSessionsPreviewPayload(ts: 0, previews: []) + } + var params: [String: AnyCodable] = ["keys": AnyCodable(resolvedKeys)] + if let limit { params["limit"] = AnyCodable(limit) } + if let maxChars { params["maxChars"] = AnyCodable(maxChars) } + let timeout = timeoutMs.map { Double($0) } + return try await self.requestDecoded( + method: .sessionsPreview, + params: params, + timeoutMs: timeout) + } + + // MARK: - Chat + + func chatHistory( + sessionKey: String, + limit: Int? = nil, + timeoutMs: Int? = nil) async throws -> OpenClawChatHistoryPayload + { + let resolvedKey = self.canonicalizeSessionKey(sessionKey) + var params: [String: AnyCodable] = ["sessionKey": AnyCodable(resolvedKey)] + if let limit { params["limit"] = AnyCodable(limit) } + let timeout = timeoutMs.map { Double($0) } + return try await self.requestDecoded( + method: .chatHistory, + params: params, + timeoutMs: timeout) + } + + func chatSend( + sessionKey: String, + message: String, + thinking: String, + idempotencyKey: String, + attachments: [OpenClawChatAttachmentPayload], + timeoutMs: Int = 30000) async throws -> OpenClawChatSendResponse + { + let resolvedKey = self.canonicalizeSessionKey(sessionKey) + var params: [String: AnyCodable] = [ + "sessionKey": AnyCodable(resolvedKey), + "message": AnyCodable(message), + "thinking": AnyCodable(thinking), + "idempotencyKey": AnyCodable(idempotencyKey), + "timeoutMs": AnyCodable(timeoutMs), + ] + + if !attachments.isEmpty { + let encoded = attachments.map { att in + [ + "type": att.type, + "mimeType": att.mimeType, + "fileName": att.fileName, + "content": att.content, + ] + } + params["attachments"] = AnyCodable(encoded) + } + + return try await self.requestDecoded( + method: .chatSend, + params: params, + timeoutMs: Double(timeoutMs)) + } + + func chatAbort(sessionKey: String, runId: String) async throws -> Bool { + let resolvedKey = self.canonicalizeSessionKey(sessionKey) + struct AbortResponse: Decodable { let ok: Bool?; let aborted: Bool? } + let res: AbortResponse = try await self.requestDecoded( + method: .chatAbort, + params: ["sessionKey": AnyCodable(resolvedKey), "runId": AnyCodable(runId)]) + return res.aborted ?? false + } + + func talkMode(enabled: Bool, phase: String? = nil) async { + var params: [String: AnyCodable] = ["enabled": AnyCodable(enabled)] + if let phase { params["phase"] = AnyCodable(phase) } + try? await self.requestVoid(method: .talkMode, params: params) + } + + // MARK: - VoiceWake + + func voiceWakeGetTriggers() async throws -> [String] { + struct VoiceWakePayload: Decodable { let triggers: [String] } + let payload: VoiceWakePayload = try await self.requestDecoded(method: .voicewakeGet) + return payload.triggers + } + + func voiceWakeSetTriggers(_ triggers: [String]) async { + do { + try await self.requestVoid( + method: .voicewakeSet, + params: ["triggers": AnyCodable(triggers)], + timeoutMs: 10000) + } catch { + // Best-effort only. + } + } + + // MARK: - Node pairing + + func nodePairApprove(requestId: String) async throws { + try await self.requestVoid( + method: .nodePairApprove, + params: ["requestId": AnyCodable(requestId)], + timeoutMs: 10000) + } + + func nodePairReject(requestId: String) async throws { + try await self.requestVoid( + method: .nodePairReject, + params: ["requestId": AnyCodable(requestId)], + timeoutMs: 10000) + } + + // MARK: - Device pairing + + func devicePairApprove(requestId: String) async throws { + try await self.requestVoid( + method: .devicePairApprove, + params: ["requestId": AnyCodable(requestId)], + timeoutMs: 10000) + } + + func devicePairReject(requestId: String) async throws { + try await self.requestVoid( + method: .devicePairReject, + params: ["requestId": AnyCodable(requestId)], + timeoutMs: 10000) + } + + // MARK: - Cron + + struct CronSchedulerStatus: Decodable, Sendable { + let enabled: Bool + let storePath: String + let jobs: Int + let nextWakeAtMs: Int? + } + + func cronStatus() async throws -> CronSchedulerStatus { + try await self.requestDecoded(method: .cronStatus) + } + + func cronList(includeDisabled: Bool = true) async throws -> [CronJob] { + let res: CronListResponse = try await self.requestDecoded( + method: .cronList, + params: ["includeDisabled": AnyCodable(includeDisabled)]) + return res.jobs + } + + func cronRuns(jobId: String, limit: Int = 200) async throws -> [CronRunLogEntry] { + let res: CronRunsResponse = try await self.requestDecoded( + method: .cronRuns, + params: ["id": AnyCodable(jobId), "limit": AnyCodable(limit)]) + return res.entries + } + + func cronRun(jobId: String, force: Bool = true) async throws { + try await self.requestVoid( + method: .cronRun, + params: [ + "id": AnyCodable(jobId), + "mode": AnyCodable(force ? "force" : "due"), + ], + timeoutMs: 20000) + } + + func cronRemove(jobId: String) async throws { + try await self.requestVoid(method: .cronRemove, params: ["id": AnyCodable(jobId)]) + } + + func cronUpdate(jobId: String, patch: [String: AnyCodable]) async throws { + try await self.requestVoid( + method: .cronUpdate, + params: ["id": AnyCodable(jobId), "patch": AnyCodable(patch)]) + } + + func cronAdd(payload: [String: AnyCodable]) async throws { + try await self.requestVoid(method: .cronAdd, params: payload) + } +} diff --git a/apps/macos/Sources/OpenClaw/GatewayConnectivityCoordinator.swift b/apps/macos/Sources/OpenClaw/GatewayConnectivityCoordinator.swift new file mode 100644 index 0000000000000000000000000000000000000000..aeb1ebb9af0642670ff41040edecd6415e5bffb7 --- /dev/null +++ b/apps/macos/Sources/OpenClaw/GatewayConnectivityCoordinator.swift @@ -0,0 +1,63 @@ +import Foundation +import Observation +import OSLog + +@MainActor +@Observable +final class GatewayConnectivityCoordinator { + static let shared = GatewayConnectivityCoordinator() + + private let logger = Logger(subsystem: "ai.openclaw", category: "gateway.connectivity") + private var endpointTask: Task? + private var lastResolvedURL: URL? + + private(set) var endpointState: GatewayEndpointState? + private(set) var resolvedURL: URL? + private(set) var resolvedMode: AppState.ConnectionMode? + private(set) var resolvedHostLabel: String? + + private init() { + self.start() + } + + func start() { + guard self.endpointTask == nil else { return } + self.endpointTask = Task { [weak self] in + guard let self else { return } + let stream = await GatewayEndpointStore.shared.subscribe() + for await state in stream { + await MainActor.run { self.handleEndpointState(state) } + } + } + } + + var localEndpointHostLabel: String? { + guard self.resolvedMode == .local, let url = self.resolvedURL else { return nil } + return Self.hostLabel(for: url) + } + + private func handleEndpointState(_ state: GatewayEndpointState) { + self.endpointState = state + switch state { + case let .ready(mode, url, _, _): + self.resolvedMode = mode + self.resolvedURL = url + self.resolvedHostLabel = Self.hostLabel(for: url) + let urlChanged = self.lastResolvedURL?.absoluteString != url.absoluteString + if urlChanged { + self.lastResolvedURL = url + Task { await ControlChannel.shared.refreshEndpoint(reason: "endpoint changed") } + } + case let .connecting(mode, _): + self.resolvedMode = mode + case let .unavailable(mode, _): + self.resolvedMode = mode + } + } + + private static func hostLabel(for url: URL) -> String { + let host = url.host ?? url.absoluteString + if let port = url.port { return "\(host):\(port)" } + return host + } +} diff --git a/apps/macos/Sources/OpenClaw/GatewayDiscoveryHelpers.swift b/apps/macos/Sources/OpenClaw/GatewayDiscoveryHelpers.swift new file mode 100644 index 0000000000000000000000000000000000000000..4becd8b13cd030ca2b9ef1b0a03dccd2806b66c8 --- /dev/null +++ b/apps/macos/Sources/OpenClaw/GatewayDiscoveryHelpers.swift @@ -0,0 +1,47 @@ +import OpenClawDiscovery +import Foundation + +enum GatewayDiscoveryHelpers { + static func sshTarget(for gateway: GatewayDiscoveryModel.DiscoveredGateway) -> String? { + let host = self.sanitizedTailnetHost(gateway.tailnetDns) ?? gateway.lanHost + guard let host = self.trimmed(host), !host.isEmpty else { return nil } + let user = NSUserName() + var target = "\(user)@\(host)" + if gateway.sshPort != 22 { + target += ":\(gateway.sshPort)" + } + return target + } + + static func directUrl(for gateway: GatewayDiscoveryModel.DiscoveredGateway) -> String? { + self.directGatewayUrl( + tailnetDns: gateway.tailnetDns, + lanHost: gateway.lanHost, + gatewayPort: gateway.gatewayPort) + } + + static func directGatewayUrl( + tailnetDns: String?, + lanHost: String?, + gatewayPort: Int?) -> String? + { + if let tailnetDns = self.sanitizedTailnetHost(tailnetDns) { + return "wss://\(tailnetDns)" + } + guard let lanHost = self.trimmed(lanHost), !lanHost.isEmpty else { return nil } + let port = gatewayPort ?? 18789 + return "ws://\(lanHost):\(port)" + } + + static func sanitizedTailnetHost(_ host: String?) -> String? { + guard let host = self.trimmed(host), !host.isEmpty else { return nil } + if host.hasSuffix(".internal.") || host.hasSuffix(".internal") { + return nil + } + return host + } + + private static func trimmed(_ value: String?) -> String? { + value?.trimmingCharacters(in: .whitespacesAndNewlines) + } +} diff --git a/apps/macos/Sources/OpenClaw/GatewayDiscoveryMenu.swift b/apps/macos/Sources/OpenClaw/GatewayDiscoveryMenu.swift new file mode 100644 index 0000000000000000000000000000000000000000..babab5866fd419fb79f5f58f0b7bc95659b5a53d --- /dev/null +++ b/apps/macos/Sources/OpenClaw/GatewayDiscoveryMenu.swift @@ -0,0 +1,139 @@ +import OpenClawDiscovery +import SwiftUI + +struct GatewayDiscoveryInlineList: View { + var discovery: GatewayDiscoveryModel + var currentTarget: String? + var currentUrl: String? + var transport: AppState.RemoteTransport + var onSelect: (GatewayDiscoveryModel.DiscoveredGateway) -> Void + @State private var hoveredGatewayID: GatewayDiscoveryModel.DiscoveredGateway.ID? + + var body: some View { + VStack(alignment: .leading, spacing: 6) { + HStack(alignment: .firstTextBaseline, spacing: 6) { + Image(systemName: "dot.radiowaves.left.and.right") + .font(.caption) + .foregroundStyle(.secondary) + Text(self.discovery.statusText) + .font(.caption) + .foregroundStyle(.secondary) + } + + if self.discovery.gateways.isEmpty { + Text("No gateways found yet.") + .font(.caption) + .foregroundStyle(.secondary) + } else { + VStack(alignment: .leading, spacing: 6) { + ForEach(self.discovery.gateways.prefix(6)) { gateway in + let display = self.displayInfo(for: gateway) + let selected = display.selected + + Button { + withAnimation(.spring(response: 0.25, dampingFraction: 0.9)) { + self.onSelect(gateway) + } + } label: { + HStack(alignment: .center, spacing: 10) { + VStack(alignment: .leading, spacing: 2) { + Text(gateway.displayName) + .font(.callout.weight(.semibold)) + .lineLimit(1) + .truncationMode(.tail) + Text(display.label) + .font(.caption.monospaced()) + .foregroundStyle(.secondary) + .lineLimit(1) + .truncationMode(.middle) + } + Spacer(minLength: 0) + if selected { + Image(systemName: "checkmark.circle.fill") + .foregroundStyle(Color.accentColor) + } else { + Image(systemName: "arrow.right.circle") + .foregroundStyle(.secondary) + } + } + .padding(.horizontal, 10) + .padding(.vertical, 8) + .frame(maxWidth: .infinity, alignment: .leading) + .background( + RoundedRectangle(cornerRadius: 10, style: .continuous) + .fill(self.rowBackground( + selected: selected, + hovered: self.hoveredGatewayID == gateway.id))) + .overlay( + RoundedRectangle(cornerRadius: 10, style: .continuous) + .strokeBorder( + selected ? Color.accentColor.opacity(0.45) : Color.clear, + lineWidth: 1)) + .contentShape(Rectangle()) + } + .buttonStyle(.plain) + .onHover { hovering in + self.hoveredGatewayID = hovering ? gateway + .id : (self.hoveredGatewayID == gateway.id ? nil : self.hoveredGatewayID) + } + } + } + .padding(10) + .background( + RoundedRectangle(cornerRadius: 10, style: .continuous) + .fill(Color(NSColor.controlBackgroundColor))) + } + } + .help(self.transport == .direct + ? "Click a discovered gateway to fill the gateway URL." + : "Click a discovered gateway to fill the SSH target.") + } + + private func displayInfo( + for gateway: GatewayDiscoveryModel.DiscoveredGateway) -> (label: String, selected: Bool) + { + switch self.transport { + case .direct: + let url = GatewayDiscoveryHelpers.directUrl(for: gateway) + let label = url ?? "Gateway pairing only" + let selected = url != nil && self.trimmed(self.currentUrl) == url + return (label, selected) + case .ssh: + let target = GatewayDiscoveryHelpers.sshTarget(for: gateway) + let label = target ?? "Gateway pairing only" + let selected = target != nil && self.trimmed(self.currentTarget) == target + return (label, selected) + } + } + + private func rowBackground(selected: Bool, hovered: Bool) -> Color { + if selected { return Color.accentColor.opacity(0.12) } + if hovered { return Color.secondary.opacity(0.08) } + return Color.clear + } + + private func trimmed(_ value: String?) -> String { + value?.trimmingCharacters(in: .whitespacesAndNewlines) ?? "" + } +} + +struct GatewayDiscoveryMenu: View { + var discovery: GatewayDiscoveryModel + var onSelect: (GatewayDiscoveryModel.DiscoveredGateway) -> Void + + var body: some View { + Menu { + if self.discovery.gateways.isEmpty { + Button(self.discovery.statusText) {} + .disabled(true) + } else { + ForEach(self.discovery.gateways) { gateway in + Button(gateway.displayName) { self.onSelect(gateway) } + } + } + } label: { + Image(systemName: "dot.radiowaves.left.and.right") + } + .help("Discover OpenClaw gateways on your LAN") + } +} diff --git a/apps/macos/Sources/OpenClaw/GatewayDiscoveryPreferences.swift b/apps/macos/Sources/OpenClaw/GatewayDiscoveryPreferences.swift new file mode 100644 index 0000000000000000000000000000000000000000..d725fdba5871c0645b0848b0ac96ac4541b2c252 --- /dev/null +++ b/apps/macos/Sources/OpenClaw/GatewayDiscoveryPreferences.swift @@ -0,0 +1,25 @@ +import Foundation + +enum GatewayDiscoveryPreferences { + private static let preferredStableIDKey = "gateway.preferredStableID" + private static let legacyPreferredStableIDKey = "bridge.preferredStableID" + + static func preferredStableID() -> String? { + let defaults = UserDefaults.standard + let raw = defaults.string(forKey: self.preferredStableIDKey) + ?? defaults.string(forKey: self.legacyPreferredStableIDKey) + let trimmed = raw?.trimmingCharacters(in: .whitespacesAndNewlines) + return trimmed?.isEmpty == false ? trimmed : nil + } + + static func setPreferredStableID(_ stableID: String?) { + let trimmed = stableID?.trimmingCharacters(in: .whitespacesAndNewlines) + if let trimmed, !trimmed.isEmpty { + UserDefaults.standard.set(trimmed, forKey: self.preferredStableIDKey) + UserDefaults.standard.removeObject(forKey: self.legacyPreferredStableIDKey) + } else { + UserDefaults.standard.removeObject(forKey: self.preferredStableIDKey) + UserDefaults.standard.removeObject(forKey: self.legacyPreferredStableIDKey) + } + } +} diff --git a/apps/macos/Sources/OpenClaw/GatewayEndpointStore.swift b/apps/macos/Sources/OpenClaw/GatewayEndpointStore.swift new file mode 100644 index 0000000000000000000000000000000000000000..20961e379bf710d095da09411666bd417be26767 --- /dev/null +++ b/apps/macos/Sources/OpenClaw/GatewayEndpointStore.swift @@ -0,0 +1,696 @@ +import ConcurrencyExtras +import Foundation +import OSLog + +enum GatewayEndpointState: Sendable, Equatable { + case ready(mode: AppState.ConnectionMode, url: URL, token: String?, password: String?) + case connecting(mode: AppState.ConnectionMode, detail: String) + case unavailable(mode: AppState.ConnectionMode, reason: String) +} + +/// Single place to resolve (and publish) the effective gateway control endpoint. +/// +/// This is intentionally separate from `GatewayConnection`: +/// - `GatewayConnection` consumes the resolved endpoint (no tunnel side-effects). +/// - The endpoint store owns observation + explicit "ensure tunnel" actions. +actor GatewayEndpointStore { + static let shared = GatewayEndpointStore() + private static let supportedBindModes: Set = [ + "loopback", + "tailnet", + "lan", + "auto", + "custom", + ] + private static let remoteConnectingDetail = "Connecting to remote gateway…" + private static let staticLogger = Logger(subsystem: "ai.openclaw", category: "gateway-endpoint") + private enum EnvOverrideWarningKind: Sendable { + case token + case password + } + + private static let envOverrideWarnings = LockIsolated((token: false, password: false)) + + struct Deps: Sendable { + let mode: @Sendable () async -> AppState.ConnectionMode + let token: @Sendable () -> String? + let password: @Sendable () -> String? + let localPort: @Sendable () -> Int + let localHost: @Sendable () async -> String + let remotePortIfRunning: @Sendable () async -> UInt16? + let ensureRemoteTunnel: @Sendable () async throws -> UInt16 + + static let live = Deps( + mode: { await MainActor.run { AppStateStore.shared.connectionMode } }, + token: { + let root = OpenClawConfigFile.loadDict() + let isRemote = ConnectionModeResolver.resolve(root: root).mode == .remote + return GatewayEndpointStore.resolveGatewayToken( + isRemote: isRemote, + root: root, + env: ProcessInfo.processInfo.environment, + launchdSnapshot: GatewayLaunchAgentManager.launchdConfigSnapshot()) + }, + password: { + let root = OpenClawConfigFile.loadDict() + let isRemote = ConnectionModeResolver.resolve(root: root).mode == .remote + return GatewayEndpointStore.resolveGatewayPassword( + isRemote: isRemote, + root: root, + env: ProcessInfo.processInfo.environment, + launchdSnapshot: GatewayLaunchAgentManager.launchdConfigSnapshot()) + }, + localPort: { GatewayEnvironment.gatewayPort() }, + localHost: { + let root = OpenClawConfigFile.loadDict() + let bind = GatewayEndpointStore.resolveGatewayBindMode( + root: root, + env: ProcessInfo.processInfo.environment) + let customBindHost = GatewayEndpointStore.resolveGatewayCustomBindHost(root: root) + let tailscaleIP = await MainActor.run { TailscaleService.shared.tailscaleIP } + ?? TailscaleService.fallbackTailnetIPv4() + return GatewayEndpointStore.resolveLocalGatewayHost( + bindMode: bind, + customBindHost: customBindHost, + tailscaleIP: tailscaleIP) + }, + remotePortIfRunning: { await RemoteTunnelManager.shared.controlTunnelPortIfRunning() }, + ensureRemoteTunnel: { try await RemoteTunnelManager.shared.ensureControlTunnel() }) + } + + private static func resolveGatewayPassword( + isRemote: Bool, + root: [String: Any], + env: [String: String], + launchdSnapshot: LaunchAgentPlistSnapshot?) -> String? + { + let raw = env["OPENCLAW_GATEWAY_PASSWORD"] ?? "" + let trimmed = raw.trimmingCharacters(in: .whitespacesAndNewlines) + if !trimmed.isEmpty { + if let configPassword = self.resolveConfigPassword(isRemote: isRemote, root: root), + !configPassword.isEmpty + { + self.warnEnvOverrideOnce( + kind: .password, + envVar: "OPENCLAW_GATEWAY_PASSWORD", + configKey: isRemote ? "gateway.remote.password" : "gateway.auth.password") + } + return trimmed + } + if isRemote { + if let gateway = root["gateway"] as? [String: Any], + let remote = gateway["remote"] as? [String: Any], + let password = remote["password"] as? String + { + let pw = password.trimmingCharacters(in: .whitespacesAndNewlines) + if !pw.isEmpty { + return pw + } + } + return nil + } + if let gateway = root["gateway"] as? [String: Any], + let auth = gateway["auth"] as? [String: Any], + let password = auth["password"] as? String + { + let pw = password.trimmingCharacters(in: .whitespacesAndNewlines) + if !pw.isEmpty { + return pw + } + } + if let password = launchdSnapshot?.password?.trimmingCharacters(in: .whitespacesAndNewlines), + !password.isEmpty + { + return password + } + return nil + } + + private static func resolveConfigPassword(isRemote: Bool, root: [String: Any]) -> String? { + if isRemote { + if let gateway = root["gateway"] as? [String: Any], + let remote = gateway["remote"] as? [String: Any], + let password = remote["password"] as? String + { + return password.trimmingCharacters(in: .whitespacesAndNewlines) + } + return nil + } + + if let gateway = root["gateway"] as? [String: Any], + let auth = gateway["auth"] as? [String: Any], + let password = auth["password"] as? String + { + return password.trimmingCharacters(in: .whitespacesAndNewlines) + } + return nil + } + + private static func resolveGatewayToken( + isRemote: Bool, + root: [String: Any], + env: [String: String], + launchdSnapshot: LaunchAgentPlistSnapshot?) -> String? + { + let raw = env["OPENCLAW_GATEWAY_TOKEN"] ?? "" + let trimmed = raw.trimmingCharacters(in: .whitespacesAndNewlines) + if !trimmed.isEmpty { + if let configToken = self.resolveConfigToken(isRemote: isRemote, root: root), + !configToken.isEmpty, + configToken != trimmed + { + self.warnEnvOverrideOnce( + kind: .token, + envVar: "OPENCLAW_GATEWAY_TOKEN", + configKey: isRemote ? "gateway.remote.token" : "gateway.auth.token") + } + return trimmed + } + + if let configToken = self.resolveConfigToken(isRemote: isRemote, root: root), + !configToken.isEmpty + { + return configToken + } + + if isRemote { + return nil + } + + if let token = launchdSnapshot?.token?.trimmingCharacters(in: .whitespacesAndNewlines), + !token.isEmpty + { + return token + } + + return nil + } + + private static func resolveConfigToken(isRemote: Bool, root: [String: Any]) -> String? { + if isRemote { + if let gateway = root["gateway"] as? [String: Any], + let remote = gateway["remote"] as? [String: Any], + let token = remote["token"] as? String + { + return token.trimmingCharacters(in: .whitespacesAndNewlines) + } + return nil + } + + if let gateway = root["gateway"] as? [String: Any], + let auth = gateway["auth"] as? [String: Any], + let token = auth["token"] as? String + { + return token.trimmingCharacters(in: .whitespacesAndNewlines) + } + return nil + } + + private static func warnEnvOverrideOnce( + kind: EnvOverrideWarningKind, + envVar: String, + configKey: String) + { + let shouldWarn = Self.envOverrideWarnings.withValue { state in + switch kind { + case .token: + guard !state.token else { return false } + state.token = true + return true + case .password: + guard !state.password else { return false } + state.password = true + return true + } + } + guard shouldWarn else { return } + Self.staticLogger.warning( + "\(envVar, privacy: .public) is set and overrides \(configKey, privacy: .public). " + + "If this is unintentional, clear it with: launchctl unsetenv \(envVar, privacy: .public)") + } + + private let deps: Deps + private let logger = Logger(subsystem: "ai.openclaw", category: "gateway-endpoint") + + private var state: GatewayEndpointState + private var subscribers: [UUID: AsyncStream.Continuation] = [:] + private var remoteEnsure: (token: UUID, task: Task)? + + init(deps: Deps = .live) { + self.deps = deps + let modeRaw = UserDefaults.standard.string(forKey: connectionModeKey) + let initialMode: AppState.ConnectionMode + if let modeRaw { + initialMode = AppState.ConnectionMode(rawValue: modeRaw) ?? .local + } else { + let seen = UserDefaults.standard.bool(forKey: "openclaw.onboardingSeen") + initialMode = seen ? .local : .unconfigured + } + + let port = deps.localPort() + let bind = GatewayEndpointStore.resolveGatewayBindMode( + root: OpenClawConfigFile.loadDict(), + env: ProcessInfo.processInfo.environment) + let customBindHost = GatewayEndpointStore.resolveGatewayCustomBindHost(root: OpenClawConfigFile.loadDict()) + let scheme = GatewayEndpointStore.resolveGatewayScheme( + root: OpenClawConfigFile.loadDict(), + env: ProcessInfo.processInfo.environment) + let host = GatewayEndpointStore.resolveLocalGatewayHost( + bindMode: bind, + customBindHost: customBindHost, + tailscaleIP: nil) + let token = deps.token() + let password = deps.password() + switch initialMode { + case .local: + self.state = .ready( + mode: .local, + url: URL(string: "\(scheme)://\(host):\(port)")!, + token: token, + password: password) + case .remote: + self.state = .connecting(mode: .remote, detail: Self.remoteConnectingDetail) + Task { await self.setMode(.remote) } + case .unconfigured: + self.state = .unavailable(mode: .unconfigured, reason: "Gateway not configured") + } + } + + func subscribe(bufferingNewest: Int = 1) -> AsyncStream { + let id = UUID() + let initial = self.state + let store = self + return AsyncStream(bufferingPolicy: .bufferingNewest(bufferingNewest)) { continuation in + continuation.yield(initial) + self.subscribers[id] = continuation + continuation.onTermination = { @Sendable _ in + Task { await store.removeSubscriber(id) } + } + } + } + + func refresh() async { + let mode = await self.deps.mode() + await self.setMode(mode) + } + + func setMode(_ mode: AppState.ConnectionMode) async { + let token = self.deps.token() + let password = self.deps.password() + switch mode { + case .local: + self.cancelRemoteEnsure() + let port = self.deps.localPort() + let host = await self.deps.localHost() + let scheme = GatewayEndpointStore.resolveGatewayScheme( + root: OpenClawConfigFile.loadDict(), + env: ProcessInfo.processInfo.environment) + self.setState(.ready( + mode: .local, + url: URL(string: "\(scheme)://\(host):\(port)")!, + token: token, + password: password)) + case .remote: + let root = OpenClawConfigFile.loadDict() + if GatewayRemoteConfig.resolveTransport(root: root) == .direct { + guard let url = GatewayRemoteConfig.resolveGatewayUrl(root: root) else { + self.cancelRemoteEnsure() + self.setState(.unavailable( + mode: .remote, + reason: "gateway.remote.url missing or invalid for direct transport")) + return + } + self.cancelRemoteEnsure() + self.setState(.ready(mode: .remote, url: url, token: token, password: password)) + return + } + let port = await self.deps.remotePortIfRunning() + guard let port else { + self.setState(.connecting(mode: .remote, detail: Self.remoteConnectingDetail)) + self.kickRemoteEnsureIfNeeded(detail: Self.remoteConnectingDetail) + return + } + self.cancelRemoteEnsure() + let scheme = GatewayEndpointStore.resolveGatewayScheme( + root: OpenClawConfigFile.loadDict(), + env: ProcessInfo.processInfo.environment) + self.setState(.ready( + mode: .remote, + url: URL(string: "\(scheme)://127.0.0.1:\(Int(port))")!, + token: token, + password: password)) + case .unconfigured: + self.cancelRemoteEnsure() + self.setState(.unavailable(mode: .unconfigured, reason: "Gateway not configured")) + } + } + + /// Explicit action: ensure the remote control tunnel is established and publish the resolved endpoint. + func ensureRemoteControlTunnel() async throws -> UInt16 { + let mode = await self.deps.mode() + guard mode == .remote else { + throw NSError( + domain: "RemoteTunnel", + code: 1, + userInfo: [NSLocalizedDescriptionKey: "Remote mode is not enabled"]) + } + let root = OpenClawConfigFile.loadDict() + if GatewayRemoteConfig.resolveTransport(root: root) == .direct { + guard let url = GatewayRemoteConfig.resolveGatewayUrl(root: root) else { + throw NSError( + domain: "GatewayEndpoint", + code: 1, + userInfo: [NSLocalizedDescriptionKey: "gateway.remote.url missing or invalid"]) + } + guard let port = GatewayRemoteConfig.defaultPort(for: url), + let portInt = UInt16(exactly: port) + else { + throw NSError( + domain: "GatewayEndpoint", + code: 1, + userInfo: [NSLocalizedDescriptionKey: "Invalid gateway.remote.url port"]) + } + self.logger.info("remote transport direct; skipping SSH tunnel") + return portInt + } + let config = try await self.ensureRemoteConfig(detail: Self.remoteConnectingDetail) + guard let portInt = config.0.port, let port = UInt16(exactly: portInt) else { + throw NSError( + domain: "GatewayEndpoint", + code: 1, + userInfo: [NSLocalizedDescriptionKey: "Missing tunnel port"]) + } + return port + } + + func requireConfig() async throws -> GatewayConnection.Config { + await self.refresh() + switch self.state { + case let .ready(_, url, token, password): + return (url, token, password) + case let .connecting(mode, _): + guard mode == .remote else { + throw NSError(domain: "GatewayEndpoint", code: 1, userInfo: [NSLocalizedDescriptionKey: "Connecting…"]) + } + return try await self.ensureRemoteConfig(detail: Self.remoteConnectingDetail) + case let .unavailable(mode, reason): + guard mode == .remote else { + throw NSError(domain: "GatewayEndpoint", code: 1, userInfo: [NSLocalizedDescriptionKey: reason]) + } + + // Auto-recover for remote mode: if the SSH control tunnel died (or hasn't been created yet), + // recreate it on demand so callers can recover without a manual reconnect. + self.logger.info( + "endpoint unavailable; ensuring remote control tunnel reason=\(reason, privacy: .public)") + return try await self.ensureRemoteConfig(detail: Self.remoteConnectingDetail) + } + } + + private func cancelRemoteEnsure() { + self.remoteEnsure?.task.cancel() + self.remoteEnsure = nil + } + + private func kickRemoteEnsureIfNeeded(detail: String) { + if self.remoteEnsure != nil { + self.setState(.connecting(mode: .remote, detail: detail)) + return + } + + let deps = self.deps + let token = UUID() + let task = Task.detached(priority: .utility) { try await deps.ensureRemoteTunnel() } + self.remoteEnsure = (token: token, task: task) + self.setState(.connecting(mode: .remote, detail: detail)) + } + + private func ensureRemoteConfig(detail: String) async throws -> GatewayConnection.Config { + let mode = await self.deps.mode() + guard mode == .remote else { + throw NSError( + domain: "RemoteTunnel", + code: 1, + userInfo: [NSLocalizedDescriptionKey: "Remote mode is not enabled"]) + } + + let root = OpenClawConfigFile.loadDict() + if GatewayRemoteConfig.resolveTransport(root: root) == .direct { + guard let url = GatewayRemoteConfig.resolveGatewayUrl(root: root) else { + throw NSError( + domain: "GatewayEndpoint", + code: 1, + userInfo: [NSLocalizedDescriptionKey: "gateway.remote.url missing or invalid"]) + } + let token = self.deps.token() + let password = self.deps.password() + self.cancelRemoteEnsure() + self.setState(.ready(mode: .remote, url: url, token: token, password: password)) + return (url, token, password) + } + + self.kickRemoteEnsureIfNeeded(detail: detail) + guard let ensure = self.remoteEnsure else { + throw NSError(domain: "GatewayEndpoint", code: 1, userInfo: [NSLocalizedDescriptionKey: "Connecting…"]) + } + + do { + let forwarded = try await ensure.task.value + let stillRemote = await self.deps.mode() == .remote + guard stillRemote else { + throw NSError( + domain: "RemoteTunnel", + code: 1, + userInfo: [NSLocalizedDescriptionKey: "Remote mode is not enabled"]) + } + + if self.remoteEnsure?.token == ensure.token { + self.remoteEnsure = nil + } + + let token = self.deps.token() + let password = self.deps.password() + let scheme = GatewayEndpointStore.resolveGatewayScheme( + root: OpenClawConfigFile.loadDict(), + env: ProcessInfo.processInfo.environment) + let url = URL(string: "\(scheme)://127.0.0.1:\(Int(forwarded))")! + self.setState(.ready(mode: .remote, url: url, token: token, password: password)) + return (url, token, password) + } catch let err as CancellationError { + if self.remoteEnsure?.token == ensure.token { + self.remoteEnsure = nil + } + throw err + } catch { + if self.remoteEnsure?.token == ensure.token { + self.remoteEnsure = nil + } + let msg = "Remote control tunnel failed (\(error.localizedDescription))" + self.setState(.unavailable(mode: .remote, reason: msg)) + self.logger.error("remote control tunnel ensure failed \(msg, privacy: .public)") + throw NSError(domain: "GatewayEndpoint", code: 1, userInfo: [NSLocalizedDescriptionKey: msg]) + } + } + + private func removeSubscriber(_ id: UUID) { + self.subscribers[id] = nil + } + + private func setState(_ next: GatewayEndpointState) { + guard next != self.state else { return } + self.state = next + for (_, continuation) in self.subscribers { + continuation.yield(next) + } + switch next { + case let .ready(mode, url, _, _): + let modeDesc = String(describing: mode) + let urlDesc = url.absoluteString + self.logger + .debug( + "resolved endpoint mode=\(modeDesc, privacy: .public) url=\(urlDesc, privacy: .public)") + case let .connecting(mode, detail): + let modeDesc = String(describing: mode) + self.logger + .debug( + "endpoint connecting mode=\(modeDesc, privacy: .public) detail=\(detail, privacy: .public)") + case let .unavailable(mode, reason): + let modeDesc = String(describing: mode) + self.logger + .debug( + "endpoint unavailable mode=\(modeDesc, privacy: .public) reason=\(reason, privacy: .public)") + } + } + + func maybeFallbackToTailnet(from currentURL: URL) async -> GatewayConnection.Config? { + let mode = await self.deps.mode() + guard mode == .local else { return nil } + + let root = OpenClawConfigFile.loadDict() + let bind = GatewayEndpointStore.resolveGatewayBindMode( + root: root, + env: ProcessInfo.processInfo.environment) + guard bind == "tailnet" else { return nil } + + let currentHost = currentURL.host?.lowercased() ?? "" + guard currentHost == "127.0.0.1" || currentHost == "localhost" else { return nil } + + let tailscaleIP = await MainActor.run { TailscaleService.shared.tailscaleIP } + ?? TailscaleService.fallbackTailnetIPv4() + guard let tailscaleIP, !tailscaleIP.isEmpty else { return nil } + + let scheme = GatewayEndpointStore.resolveGatewayScheme( + root: root, + env: ProcessInfo.processInfo.environment) + let port = self.deps.localPort() + let token = self.deps.token() + let password = self.deps.password() + let url = URL(string: "\(scheme)://\(tailscaleIP):\(port)")! + + self.logger.info("auto bind fallback to tailnet host=\(tailscaleIP, privacy: .public)") + self.setState(.ready(mode: .local, url: url, token: token, password: password)) + return (url, token, password) + } + + private static func resolveGatewayBindMode( + root: [String: Any], + env: [String: String]) -> String? + { + if let envBind = env["OPENCLAW_GATEWAY_BIND"] { + let trimmed = envBind.trimmingCharacters(in: .whitespacesAndNewlines).lowercased() + if self.supportedBindModes.contains(trimmed) { + return trimmed + } + } + if let gateway = root["gateway"] as? [String: Any], + let bind = gateway["bind"] as? String + { + let trimmed = bind.trimmingCharacters(in: .whitespacesAndNewlines).lowercased() + if self.supportedBindModes.contains(trimmed) { + return trimmed + } + } + return nil + } + + private static func resolveGatewayCustomBindHost(root: [String: Any]) -> String? { + if let gateway = root["gateway"] as? [String: Any], + let customBindHost = gateway["customBindHost"] as? String + { + let trimmed = customBindHost.trimmingCharacters(in: .whitespacesAndNewlines) + return trimmed.isEmpty ? nil : trimmed + } + return nil + } + + private static func resolveGatewayScheme( + root: [String: Any], + env: [String: String]) -> String + { + if let envValue = env["OPENCLAW_GATEWAY_TLS"]?.trimmingCharacters(in: .whitespacesAndNewlines), + !envValue.isEmpty + { + return (envValue == "1" || envValue.lowercased() == "true") ? "wss" : "ws" + } + if let gateway = root["gateway"] as? [String: Any], + let tls = gateway["tls"] as? [String: Any], + let enabled = tls["enabled"] as? Bool + { + return enabled ? "wss" : "ws" + } + return "ws" + } + + private static func resolveLocalGatewayHost( + bindMode: String?, + customBindHost: String?, + tailscaleIP: String?) -> String + { + switch bindMode { + case "tailnet": + tailscaleIP ?? "127.0.0.1" + case "auto": + "127.0.0.1" + case "custom": + customBindHost ?? "127.0.0.1" + default: + "127.0.0.1" + } + } +} + +extension GatewayEndpointStore { + static func dashboardURL(for config: GatewayConnection.Config) throws -> URL { + guard var components = URLComponents(url: config.url, resolvingAgainstBaseURL: false) else { + throw NSError(domain: "Dashboard", code: 1, userInfo: [ + NSLocalizedDescriptionKey: "Invalid gateway URL", + ]) + } + switch components.scheme?.lowercased() { + case "ws": + components.scheme = "http" + case "wss": + components.scheme = "https" + default: + components.scheme = "http" + } + components.path = "/" + var queryItems: [URLQueryItem] = [] + if let token = config.token?.trimmingCharacters(in: .whitespacesAndNewlines), + !token.isEmpty + { + queryItems.append(URLQueryItem(name: "token", value: token)) + } + if let password = config.password?.trimmingCharacters(in: .whitespacesAndNewlines), + !password.isEmpty + { + queryItems.append(URLQueryItem(name: "password", value: password)) + } + components.queryItems = queryItems.isEmpty ? nil : queryItems + guard let url = components.url else { + throw NSError(domain: "Dashboard", code: 2, userInfo: [ + NSLocalizedDescriptionKey: "Failed to build dashboard URL", + ]) + } + return url + } +} + +#if DEBUG +extension GatewayEndpointStore { + static func _testResolveGatewayPassword( + isRemote: Bool, + root: [String: Any], + env: [String: String], + launchdSnapshot: LaunchAgentPlistSnapshot? = nil) -> String? + { + self.resolveGatewayPassword(isRemote: isRemote, root: root, env: env, launchdSnapshot: launchdSnapshot) + } + + static func _testResolveGatewayToken( + isRemote: Bool, + root: [String: Any], + env: [String: String], + launchdSnapshot: LaunchAgentPlistSnapshot? = nil) -> String? + { + self.resolveGatewayToken(isRemote: isRemote, root: root, env: env, launchdSnapshot: launchdSnapshot) + } + + static func _testResolveGatewayBindMode( + root: [String: Any], + env: [String: String]) -> String? + { + self.resolveGatewayBindMode(root: root, env: env) + } + + static func _testResolveLocalGatewayHost( + bindMode: String?, + tailscaleIP: String?, + customBindHost: String? = nil) -> String + { + self.resolveLocalGatewayHost( + bindMode: bindMode, + customBindHost: customBindHost, + tailscaleIP: tailscaleIP) + } +} +#endif diff --git a/apps/macos/Sources/OpenClaw/GatewayEnvironment.swift b/apps/macos/Sources/OpenClaw/GatewayEnvironment.swift new file mode 100644 index 0000000000000000000000000000000000000000..1e10394c2d277cce4294469e503838eb8ab9b82f --- /dev/null +++ b/apps/macos/Sources/OpenClaw/GatewayEnvironment.swift @@ -0,0 +1,342 @@ +import OpenClawIPC +import Foundation +import OSLog + +// Lightweight SemVer helper (major.minor.patch only) for gateway compatibility checks. +struct Semver: Comparable, CustomStringConvertible, Sendable { + let major: Int + let minor: Int + let patch: Int + + var description: String { "\(self.major).\(self.minor).\(self.patch)" } + + static func < (lhs: Semver, rhs: Semver) -> Bool { + if lhs.major != rhs.major { return lhs.major < rhs.major } + if lhs.minor != rhs.minor { return lhs.minor < rhs.minor } + return lhs.patch < rhs.patch + } + + static func parse(_ raw: String?) -> Semver? { + guard let raw, !raw.isEmpty else { return nil } + let cleaned = raw.trimmingCharacters(in: .whitespacesAndNewlines) + .replacingOccurrences(of: "^v", with: "", options: .regularExpression) + let parts = cleaned.split(separator: ".") + guard parts.count >= 3, + let major = Int(parts[0]), + let minor = Int(parts[1]) + else { return nil } + // Strip prerelease suffix (e.g., "11-4" → "11", "5-beta.1" → "5") + let patchRaw = String(parts[2]) + guard let patchToken = patchRaw.split(whereSeparator: { $0 == "-" || $0 == "+" }).first, + let patchNumeric = Int(patchToken) + else { + return nil + } + return Semver(major: major, minor: minor, patch: patchNumeric) + } + + func compatible(with required: Semver) -> Bool { + // Same major and not older than required. + self.major == required.major && self >= required + } +} + +enum GatewayEnvironmentKind: Equatable { + case checking + case ok + case missingNode + case missingGateway + case incompatible(found: String, required: String) + case error(String) +} + +struct GatewayEnvironmentStatus: Equatable { + let kind: GatewayEnvironmentKind + let nodeVersion: String? + let gatewayVersion: String? + let requiredGateway: String? + let message: String + + static var checking: Self { + .init(kind: .checking, nodeVersion: nil, gatewayVersion: nil, requiredGateway: nil, message: "Checking…") + } +} + +struct GatewayCommandResolution { + let status: GatewayEnvironmentStatus + let command: [String]? +} + +enum GatewayEnvironment { + private static let logger = Logger(subsystem: "ai.openclaw", category: "gateway.env") + private static let supportedBindModes: Set = ["loopback", "tailnet", "lan", "auto"] + + static func gatewayPort() -> Int { + if let raw = ProcessInfo.processInfo.environment["OPENCLAW_GATEWAY_PORT"] { + let trimmed = raw.trimmingCharacters(in: .whitespacesAndNewlines) + if let parsed = Int(trimmed), parsed > 0 { return parsed } + } + if let configPort = OpenClawConfigFile.gatewayPort(), configPort > 0 { + return configPort + } + let stored = UserDefaults.standard.integer(forKey: "gatewayPort") + return stored > 0 ? stored : 18789 + } + + static func expectedGatewayVersion() -> Semver? { + Semver.parse(self.expectedGatewayVersionString()) + } + + static func expectedGatewayVersionString() -> String? { + let bundleVersion = Bundle.main.infoDictionary?["CFBundleShortVersionString"] as? String + let trimmed = bundleVersion?.trimmingCharacters(in: .whitespacesAndNewlines) + return (trimmed?.isEmpty == false) ? trimmed : nil + } + + // Exposed for tests so we can inject fake version checks without rewriting bundle metadata. + static func expectedGatewayVersion(from versionString: String?) -> Semver? { + Semver.parse(versionString) + } + + static func check() -> GatewayEnvironmentStatus { + let start = Date() + defer { + let elapsedMs = Int(Date().timeIntervalSince(start) * 1000) + if elapsedMs > 500 { + self.logger.warning("gateway env check slow (\(elapsedMs, privacy: .public)ms)") + } else { + self.logger.debug("gateway env check ok (\(elapsedMs, privacy: .public)ms)") + } + } + let expected = self.expectedGatewayVersion() + let expectedString = self.expectedGatewayVersionString() + + let projectRoot = CommandResolver.projectRoot() + let projectEntrypoint = CommandResolver.gatewayEntrypoint(in: projectRoot) + + switch RuntimeLocator.resolve(searchPaths: CommandResolver.preferredPaths()) { + case let .failure(err): + return GatewayEnvironmentStatus( + kind: .missingNode, + nodeVersion: nil, + gatewayVersion: nil, + requiredGateway: expectedString, + message: RuntimeLocator.describeFailure(err)) + case let .success(runtime): + let gatewayBin = CommandResolver.openclawExecutable() + + if gatewayBin == nil, projectEntrypoint == nil { + return GatewayEnvironmentStatus( + kind: .missingGateway, + nodeVersion: runtime.version.description, + gatewayVersion: nil, + requiredGateway: expectedString, + message: "openclaw CLI not found in PATH; install the CLI.") + } + + let installed = gatewayBin.flatMap { self.readGatewayVersion(binary: $0) } + ?? self.readLocalGatewayVersion(projectRoot: projectRoot) + + if let expected, let installed, !installed.compatible(with: expected) { + let expectedText = expectedString ?? expected.description + return GatewayEnvironmentStatus( + kind: .incompatible(found: installed.description, required: expectedText), + nodeVersion: runtime.version.description, + gatewayVersion: installed.description, + requiredGateway: expectedText, + message: """ + Gateway version \(installed.description) is incompatible with app \(expectedText); + install or update the global package. + """) + } + + let gatewayLabel = gatewayBin != nil ? "global" : "local" + let gatewayVersionText = installed?.description ?? "unknown" + // Avoid repeating "(local)" twice; if using the local entrypoint, show the path once. + let localPathHint = gatewayBin == nil && projectEntrypoint != nil + ? " (local: \(projectEntrypoint ?? "unknown"))" + : "" + let gatewayLabelText = gatewayBin != nil + ? "(\(gatewayLabel))" + : localPathHint.isEmpty ? "(\(gatewayLabel))" : localPathHint + return GatewayEnvironmentStatus( + kind: .ok, + nodeVersion: runtime.version.description, + gatewayVersion: gatewayVersionText, + requiredGateway: expectedString, + message: "Node \(runtime.version.description); gateway \(gatewayVersionText) \(gatewayLabelText)") + } + } + + static func resolveGatewayCommand() -> GatewayCommandResolution { + let start = Date() + defer { + let elapsedMs = Int(Date().timeIntervalSince(start) * 1000) + if elapsedMs > 500 { + self.logger.warning("gateway command resolve slow (\(elapsedMs, privacy: .public)ms)") + } else { + self.logger.debug("gateway command resolve ok (\(elapsedMs, privacy: .public)ms)") + } + } + let projectRoot = CommandResolver.projectRoot() + let projectEntrypoint = CommandResolver.gatewayEntrypoint(in: projectRoot) + let status = self.check() + let gatewayBin = CommandResolver.openclawExecutable() + let runtime = RuntimeLocator.resolve(searchPaths: CommandResolver.preferredPaths()) + + guard case .ok = status.kind else { + return GatewayCommandResolution(status: status, command: nil) + } + + let port = self.gatewayPort() + if let gatewayBin { + let bind = self.preferredGatewayBind() ?? "loopback" + let cmd = [gatewayBin, "gateway-daemon", "--port", "\(port)", "--bind", bind] + return GatewayCommandResolution(status: status, command: cmd) + } + + if let entry = projectEntrypoint, + case let .success(resolvedRuntime) = runtime + { + let bind = self.preferredGatewayBind() ?? "loopback" + let cmd = [resolvedRuntime.path, entry, "gateway-daemon", "--port", "\(port)", "--bind", bind] + return GatewayCommandResolution(status: status, command: cmd) + } + + return GatewayCommandResolution(status: status, command: nil) + } + + private static func preferredGatewayBind() -> String? { + if CommandResolver.connectionModeIsRemote() { + return nil + } + if let env = ProcessInfo.processInfo.environment["OPENCLAW_GATEWAY_BIND"] { + let trimmed = env.trimmingCharacters(in: .whitespacesAndNewlines).lowercased() + if self.supportedBindModes.contains(trimmed) { + return trimmed + } + } + + let root = OpenClawConfigFile.loadDict() + if let gateway = root["gateway"] as? [String: Any], + let bind = gateway["bind"] as? String + { + let trimmed = bind.trimmingCharacters(in: .whitespacesAndNewlines).lowercased() + if self.supportedBindModes.contains(trimmed) { + return trimmed + } + } + + return nil + } + + static func installGlobal(version: Semver?, statusHandler: @escaping @Sendable (String) -> Void) async { + await self.installGlobal(versionString: version?.description, statusHandler: statusHandler) + } + + static func installGlobal(versionString: String?, statusHandler: @escaping @Sendable (String) -> Void) async { + let preferred = CommandResolver.preferredPaths().joined(separator: ":") + let trimmed = versionString?.trimmingCharacters(in: .whitespacesAndNewlines) + let target: String = if let trimmed, !trimmed.isEmpty { + trimmed + } else { + "latest" + } + let npm = CommandResolver.findExecutable(named: "npm") + let pnpm = CommandResolver.findExecutable(named: "pnpm") + let bun = CommandResolver.findExecutable(named: "bun") + let (label, cmd): (String, [String]) = + if let npm { + ("npm", [npm, "install", "-g", "openclaw@\(target)"]) + } else if let pnpm { + ("pnpm", [pnpm, "add", "-g", "openclaw@\(target)"]) + } else if let bun { + ("bun", [bun, "add", "-g", "openclaw@\(target)"]) + } else { + ("npm", ["npm", "install", "-g", "openclaw@\(target)"]) + } + + statusHandler("Installing openclaw@\(target) via \(label)…") + + func summarize(_ text: String) -> String? { + let lines = text + .split(whereSeparator: \.isNewline) + .map { $0.trimmingCharacters(in: .whitespacesAndNewlines) } + .filter { !$0.isEmpty } + guard let last = lines.last else { return nil } + let normalized = last.replacingOccurrences(of: "\\s+", with: " ", options: .regularExpression) + return normalized.count > 200 ? String(normalized.prefix(199)) + "…" : normalized + } + + let response = await ShellExecutor.runDetailed(command: cmd, cwd: nil, env: ["PATH": preferred], timeout: 300) + if response.success { + statusHandler("Installed openclaw@\(target)") + } else { + if response.timedOut { + statusHandler("Install failed: timed out. Check your internet connection and try again.") + return + } + + let exit = response.exitCode.map { "exit \($0)" } ?? (response.errorMessage ?? "failed") + let detail = summarize(response.stderr) ?? summarize(response.stdout) + if let detail { + statusHandler("Install failed (\(exit)): \(detail)") + } else { + statusHandler("Install failed (\(exit))") + } + } + } + + // MARK: - Internals + + private static func readGatewayVersion(binary: String) -> Semver? { + let start = Date() + let process = Process() + process.executableURL = URL(fileURLWithPath: binary) + process.arguments = ["--version"] + process.environment = ["PATH": CommandResolver.preferredPaths().joined(separator: ":")] + + let pipe = Pipe() + process.standardOutput = pipe + process.standardError = pipe + do { + let data = try process.runAndReadToEnd(from: pipe) + let elapsedMs = Int(Date().timeIntervalSince(start) * 1000) + if elapsedMs > 500 { + self.logger.warning( + """ + gateway --version slow (\(elapsedMs, privacy: .public)ms) \ + bin=\(binary, privacy: .public) + """) + } else { + self.logger.debug( + """ + gateway --version ok (\(elapsedMs, privacy: .public)ms) \ + bin=\(binary, privacy: .public) + """) + } + let raw = String(data: data, encoding: .utf8)? + .trimmingCharacters(in: .whitespacesAndNewlines) + return Semver.parse(raw) + } catch { + let elapsedMs = Int(Date().timeIntervalSince(start) * 1000) + self.logger.error( + """ + gateway --version failed (\(elapsedMs, privacy: .public)ms) \ + bin=\(binary, privacy: .public) \ + err=\(error.localizedDescription, privacy: .public) + """) + return nil + } + } + + private static func readLocalGatewayVersion(projectRoot: URL) -> Semver? { + let pkg = projectRoot.appendingPathComponent("package.json") + guard let data = try? Data(contentsOf: pkg) else { return nil } + guard + let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any], + let version = json["version"] as? String + else { return nil } + return Semver.parse(version) + } +} diff --git a/apps/macos/Sources/OpenClaw/GatewayLaunchAgentManager.swift b/apps/macos/Sources/OpenClaw/GatewayLaunchAgentManager.swift new file mode 100644 index 0000000000000000000000000000000000000000..98743fec8b3682eb1b30a1ae9f0508a3ce80aeda --- /dev/null +++ b/apps/macos/Sources/OpenClaw/GatewayLaunchAgentManager.swift @@ -0,0 +1,204 @@ +import Foundation + +enum GatewayLaunchAgentManager { + private static let logger = Logger(subsystem: "ai.openclaw", category: "gateway.launchd") + private static let disableLaunchAgentMarker = ".openclaw/disable-launchagent" + + private static var disableLaunchAgentMarkerURL: URL { + FileManager().homeDirectoryForCurrentUser + .appendingPathComponent(self.disableLaunchAgentMarker) + } + + private static var plistURL: URL { + FileManager().homeDirectoryForCurrentUser + .appendingPathComponent("Library/LaunchAgents/\(gatewayLaunchdLabel).plist") + } + + static func isLaunchAgentWriteDisabled() -> Bool { + if FileManager().fileExists(atPath: self.disableLaunchAgentMarkerURL.path) { return true } + return false + } + + static func setLaunchAgentWriteDisabled(_ disabled: Bool) -> String? { + let marker = self.disableLaunchAgentMarkerURL + if disabled { + do { + try FileManager().createDirectory( + at: marker.deletingLastPathComponent(), + withIntermediateDirectories: true) + if !FileManager().fileExists(atPath: marker.path) { + FileManager().createFile(atPath: marker.path, contents: nil) + } + } catch { + return error.localizedDescription + } + return nil + } + + if FileManager().fileExists(atPath: marker.path) { + do { + try FileManager().removeItem(at: marker) + } catch { + return error.localizedDescription + } + } + return nil + } + + static func isLoaded() async -> Bool { + guard let loaded = await self.readDaemonLoaded() else { return false } + return loaded + } + + static func set(enabled: Bool, bundlePath: String, port: Int) async -> String? { + _ = bundlePath + guard !CommandResolver.connectionModeIsRemote() else { + self.logger.info("launchd change skipped (remote mode)") + return nil + } + if enabled, self.isLaunchAgentWriteDisabled() { + self.logger.info("launchd enable skipped (disable marker set)") + return nil + } + + if enabled { + self.logger.info("launchd enable requested via CLI port=\(port)") + return await self.runDaemonCommand([ + "install", + "--force", + "--port", + "\(port)", + "--runtime", + "node", + ]) + } + + self.logger.info("launchd disable requested via CLI") + return await self.runDaemonCommand(["uninstall"]) + } + + static func kickstart() async { + _ = await self.runDaemonCommand(["restart"], timeout: 20) + } + + static func launchdConfigSnapshot() -> LaunchAgentPlistSnapshot? { + LaunchAgentPlist.snapshot(url: self.plistURL) + } + + static func launchdGatewayLogPath() -> String { + let snapshot = self.launchdConfigSnapshot() + if let stdout = snapshot?.stdoutPath?.trimmingCharacters(in: .whitespacesAndNewlines), + !stdout.isEmpty + { + return stdout + } + if let stderr = snapshot?.stderrPath?.trimmingCharacters(in: .whitespacesAndNewlines), + !stderr.isEmpty + { + return stderr + } + return LogLocator.launchdGatewayLogPath + } +} + +extension GatewayLaunchAgentManager { + private static func readDaemonLoaded() async -> Bool? { + let result = await self.runDaemonCommandResult( + ["status", "--json", "--no-probe"], + timeout: 15, + quiet: true) + guard result.success, let payload = result.payload else { return nil } + guard + let json = try? JSONSerialization.jsonObject(with: payload) as? [String: Any], + let service = json["service"] as? [String: Any], + let loaded = service["loaded"] as? Bool + else { + return nil + } + return loaded + } + + private struct CommandResult { + let success: Bool + let payload: Data? + let message: String? + } + + private struct ParsedDaemonJson { + let text: String + let object: [String: Any] + } + + private static func runDaemonCommand( + _ args: [String], + timeout: Double = 15, + quiet: Bool = false) async -> String? + { + let result = await self.runDaemonCommandResult(args, timeout: timeout, quiet: quiet) + if result.success { return nil } + return result.message ?? "Gateway daemon command failed" + } + + private static func runDaemonCommandResult( + _ args: [String], + timeout: Double, + quiet: Bool) async -> CommandResult + { + let command = CommandResolver.openclawCommand( + subcommand: "gateway", + extraArgs: self.withJsonFlag(args), + // Launchd management must always run locally, even if remote mode is configured. + configRoot: ["gateway": ["mode": "local"]]) + var env = ProcessInfo.processInfo.environment + env["PATH"] = CommandResolver.preferredPaths().joined(separator: ":") + let response = await ShellExecutor.runDetailed(command: command, cwd: nil, env: env, timeout: timeout) + let parsed = self.parseDaemonJson(from: response.stdout) ?? self.parseDaemonJson(from: response.stderr) + let ok = parsed?.object["ok"] as? Bool + let message = (parsed?.object["error"] as? String) ?? (parsed?.object["message"] as? String) + let payload = parsed?.text.data(using: .utf8) + ?? (response.stdout.isEmpty ? response.stderr : response.stdout).data(using: .utf8) + let success = ok ?? response.success + if success { + return CommandResult(success: true, payload: payload, message: nil) + } + + if quiet { + return CommandResult(success: false, payload: payload, message: message) + } + + let detail = message ?? self.summarize(response.stderr) ?? self.summarize(response.stdout) + let exit = response.exitCode.map { "exit \($0)" } ?? (response.errorMessage ?? "failed") + let fullMessage = detail.map { "Gateway daemon command failed (\(exit)): \($0)" } + ?? "Gateway daemon command failed (\(exit))" + self.logger.error("\(fullMessage, privacy: .public)") + return CommandResult(success: false, payload: payload, message: detail) + } + + private static func withJsonFlag(_ args: [String]) -> [String] { + if args.contains("--json") { return args } + return args + ["--json"] + } + + private static func parseDaemonJson(from raw: String) -> ParsedDaemonJson? { + let trimmed = raw.trimmingCharacters(in: .whitespacesAndNewlines) + guard let start = trimmed.firstIndex(of: "{"), + let end = trimmed.lastIndex(of: "}") + else { + return nil + } + let jsonText = String(trimmed[start...end]) + guard let data = jsonText.data(using: .utf8) else { return nil } + guard let object = try? JSONSerialization.jsonObject(with: data) as? [String: Any] else { return nil } + return ParsedDaemonJson(text: jsonText, object: object) + } + + private static func summarize(_ text: String) -> String? { + let lines = text + .split(whereSeparator: \.isNewline) + .map { $0.trimmingCharacters(in: .whitespacesAndNewlines) } + .filter { !$0.isEmpty } + guard let last = lines.last else { return nil } + let normalized = last.replacingOccurrences(of: "\\s+", with: " ", options: .regularExpression) + return normalized.count > 200 ? String(normalized.prefix(199)) + "…" : normalized + } +} diff --git a/apps/macos/Sources/OpenClaw/GatewayProcessManager.swift b/apps/macos/Sources/OpenClaw/GatewayProcessManager.swift new file mode 100644 index 0000000000000000000000000000000000000000..e3d5263e9bc8ddf5e04a015bbadf0984b067eab8 --- /dev/null +++ b/apps/macos/Sources/OpenClaw/GatewayProcessManager.swift @@ -0,0 +1,432 @@ +import Foundation +import Observation + +@MainActor +@Observable +final class GatewayProcessManager { + static let shared = GatewayProcessManager() + + enum Status: Equatable { + case stopped + case starting + case running(details: String?) + case attachedExisting(details: String?) + case failed(String) + + var label: String { + switch self { + case .stopped: return "Stopped" + case .starting: return "Starting…" + case let .running(details): + if let details, !details.isEmpty { return "Running (\(details))" } + return "Running" + case let .attachedExisting(details): + if let details, !details.isEmpty { + return "Using existing gateway (\(details))" + } + return "Using existing gateway" + case let .failed(reason): return "Failed: \(reason)" + } + } + } + + private(set) var status: Status = .stopped { + didSet { CanvasManager.shared.refreshDebugStatus() } + } + + private(set) var log: String = "" + private(set) var environmentStatus: GatewayEnvironmentStatus = .checking + private(set) var existingGatewayDetails: String? + private(set) var lastFailureReason: String? + private var desiredActive = false + private var environmentRefreshTask: Task? + private var lastEnvironmentRefresh: Date? + private var logRefreshTask: Task? + #if DEBUG + private var testingConnection: GatewayConnection? + #endif + private let logger = Logger(subsystem: "ai.openclaw", category: "gateway.process") + + private let logLimit = 20000 // characters to keep in-memory + private let environmentRefreshMinInterval: TimeInterval = 30 + private var connection: GatewayConnection { + #if DEBUG + return self.testingConnection ?? .shared + #else + return .shared + #endif + } + + func setActive(_ active: Bool) { + // Remote mode should never spawn a local gateway; treat as stopped. + if CommandResolver.connectionModeIsRemote() { + self.desiredActive = false + self.stop() + self.status = .stopped + self.appendLog("[gateway] remote mode active; skipping local gateway\n") + self.logger.info("gateway process skipped: remote mode active") + return + } + self.logger.debug("gateway active requested active=\(active)") + self.desiredActive = active + self.refreshEnvironmentStatus() + if active { + self.startIfNeeded() + } else { + self.stop() + } + } + + func ensureLaunchAgentEnabledIfNeeded() async { + guard !CommandResolver.connectionModeIsRemote() else { return } + if GatewayLaunchAgentManager.isLaunchAgentWriteDisabled() { + self.appendLog("[gateway] launchd auto-enable skipped (attach-only)\n") + self.logger.info("gateway launchd auto-enable skipped (disable marker set)") + return + } + let enabled = await GatewayLaunchAgentManager.isLoaded() + guard !enabled else { return } + let bundlePath = Bundle.main.bundleURL.path + let port = GatewayEnvironment.gatewayPort() + self.appendLog("[gateway] auto-enabling launchd job (\(gatewayLaunchdLabel)) on port \(port)\n") + let err = await GatewayLaunchAgentManager.set(enabled: true, bundlePath: bundlePath, port: port) + if let err { + self.appendLog("[gateway] launchd auto-enable failed: \(err)\n") + } + } + + func startIfNeeded() { + guard self.desiredActive else { return } + // Do not spawn in remote mode (the gateway should run on the remote host). + guard !CommandResolver.connectionModeIsRemote() else { + self.status = .stopped + return + } + // Many surfaces can call `setActive(true)` in quick succession (startup, Canvas, health checks). + // Avoid spawning multiple concurrent "start" tasks that can thrash launchd and flap the port. + switch self.status { + case .starting, .running, .attachedExisting: + return + case .stopped, .failed: + break + } + self.status = .starting + self.logger.debug("gateway start requested") + + // First try to latch onto an already-running gateway to avoid spawning a duplicate. + Task { [weak self] in + guard let self else { return } + if await self.attachExistingGatewayIfAvailable() { + return + } + await self.enableLaunchdGateway() + } + } + + func stop() { + self.desiredActive = false + self.existingGatewayDetails = nil + self.lastFailureReason = nil + self.status = .stopped + self.logger.info("gateway stop requested") + if CommandResolver.connectionModeIsRemote() { + return + } + let bundlePath = Bundle.main.bundleURL.path + Task { + _ = await GatewayLaunchAgentManager.set( + enabled: false, + bundlePath: bundlePath, + port: GatewayEnvironment.gatewayPort()) + } + } + + func clearLastFailure() { + self.lastFailureReason = nil + } + + func refreshEnvironmentStatus(force: Bool = false) { + let now = Date() + if !force { + if self.environmentRefreshTask != nil { return } + if let last = self.lastEnvironmentRefresh, + now.timeIntervalSince(last) < self.environmentRefreshMinInterval + { + return + } + } + self.lastEnvironmentRefresh = now + self.environmentRefreshTask = Task { [weak self] in + let status = await Task.detached(priority: .utility) { + GatewayEnvironment.check() + }.value + await MainActor.run { + guard let self else { return } + self.environmentStatus = status + self.environmentRefreshTask = nil + } + } + } + + func refreshLog() { + guard self.logRefreshTask == nil else { return } + let path = GatewayLaunchAgentManager.launchdGatewayLogPath() + let limit = self.logLimit + self.logRefreshTask = Task { [weak self] in + let log = await Task.detached(priority: .utility) { + Self.readGatewayLog(path: path, limit: limit) + }.value + await MainActor.run { + guard let self else { return } + if !log.isEmpty { + self.log = log + } + self.logRefreshTask = nil + } + } + } + + // MARK: - Internals + + /// Attempt to connect to an already-running gateway on the configured port. + /// If successful, mark status as attached and skip spawning a new process. + private func attachExistingGatewayIfAvailable() async -> Bool { + let port = GatewayEnvironment.gatewayPort() + let instance = await PortGuardian.shared.describe(port: port) + let instanceText = instance.map { self.describe(instance: $0) } + let hasListener = instance != nil + + let attemptAttach = { + try await self.connection.requestRaw(method: .health, timeoutMs: 2000) + } + + for attempt in 0..<(hasListener ? 3 : 1) { + do { + let data = try await attemptAttach() + let snap = decodeHealthSnapshot(from: data) + let details = self.describe(details: instanceText, port: port, snap: snap) + self.existingGatewayDetails = details + self.clearLastFailure() + self.status = .attachedExisting(details: details) + self.appendLog("[gateway] using existing instance: \(details)\n") + self.logger.info("gateway using existing instance details=\(details)") + self.refreshControlChannelIfNeeded(reason: "attach existing") + self.refreshLog() + return true + } catch { + if attempt < 2, hasListener { + try? await Task.sleep(nanoseconds: 250_000_000) + continue + } + + if hasListener { + let reason = self.describeAttachFailure(error, port: port, instance: instance) + self.existingGatewayDetails = instanceText + self.status = .failed(reason) + self.lastFailureReason = reason + self.appendLog("[gateway] existing listener on port \(port) but attach failed: \(reason)\n") + self.logger.warning("gateway attach failed reason=\(reason)") + return true + } + + // No reachable gateway (and no listener) — fall through to spawn. + self.existingGatewayDetails = nil + return false + } + } + + self.existingGatewayDetails = nil + return false + } + + private func describe(details instance: String?, port: Int, snap: HealthSnapshot?) -> String { + let instanceText = instance ?? "pid unknown" + if let snap { + let order = snap.channelOrder ?? Array(snap.channels.keys) + let linkId = order.first(where: { snap.channels[$0]?.linked == true }) + ?? order.first(where: { snap.channels[$0]?.linked != nil }) + guard let linkId else { + return "port \(port), health probe succeeded, \(instanceText)" + } + let linked = snap.channels[linkId]?.linked ?? false + let authAge = snap.channels[linkId]?.authAgeMs.flatMap(msToAge) ?? "unknown age" + let label = + snap.channelLabels?[linkId] ?? + linkId.capitalized + let linkText = linked ? "linked" : "not linked" + return "port \(port), \(label) \(linkText), auth \(authAge), \(instanceText)" + } + return "port \(port), health probe succeeded, \(instanceText)" + } + + private func describe(instance: PortGuardian.Descriptor) -> String { + let path = instance.executablePath ?? "path unknown" + return "pid \(instance.pid) \(instance.command) @ \(path)" + } + + private func describeAttachFailure(_ error: Error, port: Int, instance: PortGuardian.Descriptor?) -> String { + let ns = error as NSError + let message = ns.localizedDescription.isEmpty ? "unknown error" : ns.localizedDescription + let lower = message.lowercased() + if self.isGatewayAuthFailure(error) { + return """ + Gateway on port \(port) rejected auth. Set gateway.auth.token to match the running gateway \ + (or clear it on the gateway) and retry. + """ + } + if lower.contains("protocol mismatch") { + return "Gateway on port \(port) is incompatible (protocol mismatch). Update the app/gateway." + } + if lower.contains("unexpected response") || lower.contains("invalid response") { + return "Port \(port) returned non-gateway data; another process is using it." + } + if let instance { + let instanceText = self.describe(instance: instance) + return "Gateway listener found on port \(port) (\(instanceText)) but health check failed: \(message)" + } + return "Gateway listener found on port \(port) but health check failed: \(message)" + } + + private func isGatewayAuthFailure(_ error: Error) -> Bool { + if let urlError = error as? URLError, urlError.code == .dataNotAllowed { + return true + } + let ns = error as NSError + if ns.domain == "Gateway", ns.code == 1008 { return true } + let lower = ns.localizedDescription.lowercased() + return lower.contains("unauthorized") || lower.contains("auth") + } + + private func enableLaunchdGateway() async { + self.existingGatewayDetails = nil + let resolution = await Task.detached(priority: .utility) { + GatewayEnvironment.resolveGatewayCommand() + }.value + await MainActor.run { self.environmentStatus = resolution.status } + guard resolution.command != nil else { + await MainActor.run { + self.status = .failed(resolution.status.message) + } + self.logger.error("gateway command resolve failed: \(resolution.status.message)") + return + } + + if GatewayLaunchAgentManager.isLaunchAgentWriteDisabled() { + let message = "Launchd disabled; start the Gateway manually or disable attach-only." + self.status = .failed(message) + self.lastFailureReason = "launchd disabled" + self.appendLog("[gateway] launchd disabled; skipping auto-start\n") + self.logger.info("gateway launchd enable skipped (disable marker set)") + return + } + + let bundlePath = Bundle.main.bundleURL.path + let port = GatewayEnvironment.gatewayPort() + self.appendLog("[gateway] enabling launchd job (\(gatewayLaunchdLabel)) on port \(port)\n") + self.logger.info("gateway enabling launchd port=\(port)") + let err = await GatewayLaunchAgentManager.set(enabled: true, bundlePath: bundlePath, port: port) + if let err { + self.status = .failed(err) + self.lastFailureReason = err + self.logger.error("gateway launchd enable failed: \(err)") + return + } + + // Best-effort: wait for the gateway to accept connections. + let deadline = Date().addingTimeInterval(6) + while Date() < deadline { + if !self.desiredActive { return } + do { + _ = try await self.connection.requestRaw(method: .health, timeoutMs: 1500) + let instance = await PortGuardian.shared.describe(port: port) + let details = instance.map { "pid \($0.pid)" } + self.clearLastFailure() + self.status = .running(details: details) + self.logger.info("gateway started details=\(details ?? "ok")") + self.refreshControlChannelIfNeeded(reason: "gateway started") + self.refreshLog() + return + } catch { + try? await Task.sleep(nanoseconds: 400_000_000) + } + } + + self.status = .failed("Gateway did not start in time") + self.lastFailureReason = "launchd start timeout" + self.logger.warning("gateway start timed out") + } + + private func appendLog(_ chunk: String) { + self.log.append(chunk) + if self.log.count > self.logLimit { + self.log = String(self.log.suffix(self.logLimit)) + } + } + + private func refreshControlChannelIfNeeded(reason: String) { + switch ControlChannel.shared.state { + case .connected, .connecting: + return + case .disconnected, .degraded: + break + } + self.appendLog("[gateway] refreshing control channel (\(reason))\n") + self.logger.debug("gateway control channel refresh reason=\(reason)") + Task { await ControlChannel.shared.configure() } + } + + func waitForGatewayReady(timeout: TimeInterval = 6) async -> Bool { + let deadline = Date().addingTimeInterval(timeout) + while Date() < deadline { + if !self.desiredActive { return false } + do { + _ = try await self.connection.requestRaw(method: .health, timeoutMs: 1500) + self.clearLastFailure() + return true + } catch { + try? await Task.sleep(nanoseconds: 300_000_000) + } + } + self.appendLog("[gateway] readiness wait timed out\n") + self.logger.warning("gateway readiness wait timed out") + return false + } + + func clearLog() { + self.log = "" + try? FileManager().removeItem(atPath: GatewayLaunchAgentManager.launchdGatewayLogPath()) + self.logger.debug("gateway log cleared") + } + + func setProjectRoot(path: String) { + CommandResolver.setProjectRoot(path) + } + + func projectRootPath() -> String { + CommandResolver.projectRootPath() + } + + private nonisolated static func readGatewayLog(path: String, limit: Int) -> String { + guard FileManager().fileExists(atPath: path) else { return "" } + guard let data = try? Data(contentsOf: URL(fileURLWithPath: path)) else { return "" } + let text = String(data: data, encoding: .utf8) ?? "" + if text.count <= limit { return text } + return String(text.suffix(limit)) + } +} + +#if DEBUG +extension GatewayProcessManager { + func setTestingConnection(_ connection: GatewayConnection?) { + self.testingConnection = connection + } + + func setTestingDesiredActive(_ active: Bool) { + self.desiredActive = active + } + + func setTestingLastFailureReason(_ reason: String?) { + self.lastFailureReason = reason + } +} +#endif diff --git a/apps/macos/Sources/OpenClaw/GatewayRemoteConfig.swift b/apps/macos/Sources/OpenClaw/GatewayRemoteConfig.swift new file mode 100644 index 0000000000000000000000000000000000000000..0b8ab35159d2e60ccb4d5acbc99fa161f7e95a20 --- /dev/null +++ b/apps/macos/Sources/OpenClaw/GatewayRemoteConfig.swift @@ -0,0 +1,64 @@ +import Foundation + +enum GatewayRemoteConfig { + static func resolveTransport(root: [String: Any]) -> AppState.RemoteTransport { + guard let gateway = root["gateway"] as? [String: Any], + let remote = gateway["remote"] as? [String: Any], + let raw = remote["transport"] as? String + else { + return .ssh + } + let trimmed = raw.trimmingCharacters(in: .whitespacesAndNewlines).lowercased() + return trimmed == AppState.RemoteTransport.direct.rawValue ? .direct : .ssh + } + + static func resolveUrlString(root: [String: Any]) -> String? { + guard let gateway = root["gateway"] as? [String: Any], + let remote = gateway["remote"] as? [String: Any], + let urlRaw = remote["url"] as? String + else { + return nil + } + let trimmed = urlRaw.trimmingCharacters(in: .whitespacesAndNewlines) + return trimmed.isEmpty ? nil : trimmed + } + + static func resolveGatewayUrl(root: [String: Any]) -> URL? { + guard let raw = self.resolveUrlString(root: root) else { return nil } + return self.normalizeGatewayUrl(raw) + } + + static func normalizeGatewayUrlString(_ raw: String) -> String? { + self.normalizeGatewayUrl(raw)?.absoluteString + } + + static func normalizeGatewayUrl(_ raw: String) -> URL? { + let trimmed = raw.trimmingCharacters(in: .whitespacesAndNewlines) + guard !trimmed.isEmpty, let url = URL(string: trimmed) else { return nil } + let scheme = url.scheme?.lowercased() ?? "" + guard scheme == "ws" || scheme == "wss" else { return nil } + let host = url.host?.trimmingCharacters(in: .whitespacesAndNewlines) ?? "" + guard !host.isEmpty else { return nil } + if scheme == "ws", url.port == nil { + guard var components = URLComponents(url: url, resolvingAgainstBaseURL: false) else { + return url + } + components.port = 18789 + return components.url + } + return url + } + + static func defaultPort(for url: URL) -> Int? { + if let port = url.port { return port } + let scheme = url.scheme?.lowercased() ?? "" + switch scheme { + case "wss": + return 443 + case "ws": + return 18789 + default: + return nil + } + } +} diff --git a/apps/macos/Sources/OpenClaw/GeneralSettings.swift b/apps/macos/Sources/OpenClaw/GeneralSettings.swift new file mode 100644 index 0000000000000000000000000000000000000000..03855b7698af18cd49f461df3c0c31fda45018ce --- /dev/null +++ b/apps/macos/Sources/OpenClaw/GeneralSettings.swift @@ -0,0 +1,741 @@ +import AppKit +import OpenClawDiscovery +import OpenClawIPC +import OpenClawKit +import Observation +import SwiftUI + +struct GeneralSettings: View { + @Bindable var state: AppState + @AppStorage(cameraEnabledKey) private var cameraEnabled: Bool = false + private let healthStore = HealthStore.shared + private let gatewayManager = GatewayProcessManager.shared + @State private var gatewayDiscovery = GatewayDiscoveryModel( + localDisplayName: InstanceIdentity.displayName) + @State private var gatewayStatus: GatewayEnvironmentStatus = .checking + @State private var remoteStatus: RemoteStatus = .idle + @State private var showRemoteAdvanced = false + private let isPreview = ProcessInfo.processInfo.isPreview + private var isNixMode: Bool { ProcessInfo.processInfo.isNixMode } + private var remoteLabelWidth: CGFloat { 88 } + + var body: some View { + ScrollView(.vertical) { + VStack(alignment: .leading, spacing: 18) { + VStack(alignment: .leading, spacing: 12) { + SettingsToggleRow( + title: "OpenClaw active", + subtitle: "Pause to stop the OpenClaw gateway; no messages will be processed.", + binding: self.activeBinding) + + self.connectionSection + + Divider() + + SettingsToggleRow( + title: "Launch at login", + subtitle: "Automatically start OpenClaw after you sign in.", + binding: self.$state.launchAtLogin) + + SettingsToggleRow( + title: "Show Dock icon", + subtitle: "Keep OpenClaw visible in the Dock instead of menu-bar-only mode.", + binding: self.$state.showDockIcon) + + SettingsToggleRow( + title: "Play menu bar icon animations", + subtitle: "Enable idle blinks and wiggles on the status icon.", + binding: self.$state.iconAnimationsEnabled) + + SettingsToggleRow( + title: "Allow Canvas", + subtitle: "Allow the agent to show and control the Canvas panel.", + binding: self.$state.canvasEnabled) + + SettingsToggleRow( + title: "Allow Camera", + subtitle: "Allow the agent to capture a photo or short video via the built-in camera.", + binding: self.$cameraEnabled) + + SettingsToggleRow( + title: "Enable Peekaboo Bridge", + subtitle: "Allow signed tools (e.g. `peekaboo`) to drive UI automation via PeekabooBridge.", + binding: self.$state.peekabooBridgeEnabled) + + SettingsToggleRow( + title: "Enable debug tools", + subtitle: "Show the Debug tab with development utilities.", + binding: self.$state.debugPaneEnabled) + } + + Spacer(minLength: 12) + HStack { + Spacer() + Button("Quit OpenClaw") { NSApp.terminate(nil) } + .buttonStyle(.borderedProminent) + } + } + .frame(maxWidth: .infinity, alignment: .leading) + .padding(.horizontal, 22) + .padding(.bottom, 16) + } + .onAppear { + guard !self.isPreview else { return } + self.refreshGatewayStatus() + } + .onChange(of: self.state.canvasEnabled) { _, enabled in + if !enabled { + CanvasManager.shared.hideAll() + } + } + } + + private var activeBinding: Binding { + Binding( + get: { !self.state.isPaused }, + set: { self.state.isPaused = !$0 }) + } + + private var connectionSection: some View { + VStack(alignment: .leading, spacing: 10) { + Text("OpenClaw runs") + .font(.title3.weight(.semibold)) + .frame(maxWidth: .infinity, alignment: .leading) + + Picker("Mode", selection: self.$state.connectionMode) { + Text("Not configured").tag(AppState.ConnectionMode.unconfigured) + Text("Local (this Mac)").tag(AppState.ConnectionMode.local) + Text("Remote (another host)").tag(AppState.ConnectionMode.remote) + } + .pickerStyle(.menu) + .labelsHidden() + .frame(width: 260, alignment: .leading) + + if self.state.connectionMode == .unconfigured { + Text("Pick Local or Remote to start the Gateway.") + .font(.footnote) + .foregroundStyle(.secondary) + .fixedSize(horizontal: false, vertical: true) + } + + if self.state.connectionMode == .local { + // In Nix mode, gateway is managed declaratively - no install buttons. + if !self.isNixMode { + self.gatewayInstallerCard + } + TailscaleIntegrationSection( + connectionMode: self.state.connectionMode, + isPaused: self.state.isPaused) + self.healthRow + } + + if self.state.connectionMode == .remote { + self.remoteCard + } + } + } + + private var remoteCard: some View { + VStack(alignment: .leading, spacing: 10) { + self.remoteTransportRow + + if self.state.remoteTransport == .ssh { + self.remoteSshRow + } else { + self.remoteDirectRow + } + + GatewayDiscoveryInlineList( + discovery: self.gatewayDiscovery, + currentTarget: self.state.remoteTarget, + currentUrl: self.state.remoteUrl, + transport: self.state.remoteTransport) + { gateway in + self.applyDiscoveredGateway(gateway) + } + .padding(.leading, self.remoteLabelWidth + 10) + + self.remoteStatusView + .padding(.leading, self.remoteLabelWidth + 10) + + if self.state.remoteTransport == .ssh { + DisclosureGroup(isExpanded: self.$showRemoteAdvanced) { + VStack(alignment: .leading, spacing: 8) { + LabeledContent("Identity file") { + TextField("/Users/you/.ssh/id_ed25519", text: self.$state.remoteIdentity) + .textFieldStyle(.roundedBorder) + .frame(width: 280) + } + LabeledContent("Project root") { + TextField("/home/you/Projects/openclaw", text: self.$state.remoteProjectRoot) + .textFieldStyle(.roundedBorder) + .frame(width: 280) + } + LabeledContent("CLI path") { + TextField("/Applications/OpenClaw.app/.../openclaw", text: self.$state.remoteCliPath) + .textFieldStyle(.roundedBorder) + .frame(width: 280) + } + } + .padding(.top, 4) + } label: { + Text("Advanced") + .font(.callout.weight(.semibold)) + } + } + + // Diagnostics + VStack(alignment: .leading, spacing: 4) { + Text("Control channel") + .font(.caption.weight(.semibold)) + if !self.isControlStatusDuplicate || ControlChannel.shared.lastPingMs != nil { + let status = self.isControlStatusDuplicate ? nil : self.controlStatusLine + let ping = ControlChannel.shared.lastPingMs.map { "Ping \(Int($0)) ms" } + let line = [status, ping].compactMap(\.self).joined(separator: " · ") + if !line.isEmpty { + Text(line) + .font(.caption) + .foregroundStyle(.secondary) + } + } + if let hb = HeartbeatStore.shared.lastEvent { + let ageText = age(from: Date(timeIntervalSince1970: hb.ts / 1000)) + Text("Last heartbeat: \(hb.status) · \(ageText)") + .font(.caption) + .foregroundStyle(.secondary) + } + if let authLabel = ControlChannel.shared.authSourceLabel { + Text(authLabel) + .font(.caption) + .foregroundStyle(.secondary) + } + } + + if self.state.remoteTransport == .ssh { + Text("Tip: enable Tailscale for stable remote access.") + .font(.footnote) + .foregroundStyle(.secondary) + .lineLimit(1) + } else { + Text("Tip: use Tailscale Serve so the gateway has a valid HTTPS cert.") + .font(.footnote) + .foregroundStyle(.secondary) + .lineLimit(2) + } + } + .transition(.opacity) + .onAppear { self.gatewayDiscovery.start() } + .onDisappear { self.gatewayDiscovery.stop() } + } + + private var remoteTransportRow: some View { + HStack(alignment: .center, spacing: 10) { + Text("Transport") + .font(.callout.weight(.semibold)) + .frame(width: self.remoteLabelWidth, alignment: .leading) + Picker("Transport", selection: self.$state.remoteTransport) { + Text("SSH tunnel").tag(AppState.RemoteTransport.ssh) + Text("Direct (ws/wss)").tag(AppState.RemoteTransport.direct) + } + .pickerStyle(.segmented) + .frame(maxWidth: 320) + } + } + + private var remoteSshRow: some View { + let trimmedTarget = self.state.remoteTarget.trimmingCharacters(in: .whitespacesAndNewlines) + let validationMessage = CommandResolver.sshTargetValidationMessage(trimmedTarget) + let canTest = !trimmedTarget.isEmpty && validationMessage == nil + + return VStack(alignment: .leading, spacing: 4) { + HStack(alignment: .center, spacing: 10) { + Text("SSH target") + .font(.callout.weight(.semibold)) + .frame(width: self.remoteLabelWidth, alignment: .leading) + TextField("user@host[:22]", text: self.$state.remoteTarget) + .textFieldStyle(.roundedBorder) + .frame(maxWidth: .infinity) + Button { + Task { await self.testRemote() } + } label: { + if self.remoteStatus == .checking { + ProgressView().controlSize(.small) + } else { + Text("Test remote") + } + } + .buttonStyle(.borderedProminent) + .disabled(self.remoteStatus == .checking || !canTest) + } + if let validationMessage { + Text(validationMessage) + .font(.caption) + .foregroundStyle(.red) + .padding(.leading, self.remoteLabelWidth + 10) + } + } + } + + private var remoteDirectRow: some View { + VStack(alignment: .leading, spacing: 6) { + HStack(alignment: .center, spacing: 10) { + Text("Gateway") + .font(.callout.weight(.semibold)) + .frame(width: self.remoteLabelWidth, alignment: .leading) + TextField("wss://gateway.example.ts.net", text: self.$state.remoteUrl) + .textFieldStyle(.roundedBorder) + .frame(maxWidth: .infinity) + Button { + Task { await self.testRemote() } + } label: { + if self.remoteStatus == .checking { + ProgressView().controlSize(.small) + } else { + Text("Test remote") + } + } + .buttonStyle(.borderedProminent) + .disabled(self.remoteStatus == .checking || self.state.remoteUrl + .trimmingCharacters(in: .whitespacesAndNewlines).isEmpty) + } + Text("Direct mode requires a ws:// or wss:// URL (Tailscale Serve uses wss://).") + .font(.caption) + .foregroundStyle(.secondary) + .padding(.leading, self.remoteLabelWidth + 10) + } + } + + private var controlStatusLine: String { + switch ControlChannel.shared.state { + case .connected: "Connected" + case .connecting: "Connecting…" + case .disconnected: "Disconnected" + case let .degraded(msg): msg + } + } + + @ViewBuilder + private var remoteStatusView: some View { + switch self.remoteStatus { + case .idle: + EmptyView() + case .checking: + Text("Testing…") + .font(.caption) + .foregroundStyle(.secondary) + case .ok: + Label("Ready", systemImage: "checkmark.circle.fill") + .font(.caption) + .foregroundStyle(.green) + case let .failed(message): + Text(message) + .font(.caption) + .foregroundStyle(.secondary) + .lineLimit(2) + } + } + + private var isControlStatusDuplicate: Bool { + guard case let .failed(message) = self.remoteStatus else { return false } + return message == self.controlStatusLine + } + + private var gatewayInstallerCard: some View { + VStack(alignment: .leading, spacing: 8) { + HStack(spacing: 10) { + Circle() + .fill(self.gatewayStatusColor) + .frame(width: 10, height: 10) + Text(self.gatewayStatus.message) + .font(.callout) + .frame(maxWidth: .infinity, alignment: .leading) + } + + if let gatewayVersion = self.gatewayStatus.gatewayVersion, + let required = self.gatewayStatus.requiredGateway, + gatewayVersion != required + { + Text("Installed: \(gatewayVersion) · Required: \(required)") + .font(.caption) + .foregroundStyle(.secondary) + } else if let gatewayVersion = self.gatewayStatus.gatewayVersion { + Text("Gateway \(gatewayVersion) detected") + .font(.caption) + .foregroundStyle(.secondary) + } + + if let node = self.gatewayStatus.nodeVersion { + Text("Node \(node)") + .font(.caption) + .foregroundStyle(.secondary) + } + + if case let .attachedExisting(details) = self.gatewayManager.status { + Text(details ?? "Using existing gateway instance") + .font(.caption) + .foregroundStyle(.secondary) + } + + if let failure = self.gatewayManager.lastFailureReason { + Text("Last failure: \(failure)") + .font(.caption) + .foregroundStyle(.red) + } + + Button("Recheck") { self.refreshGatewayStatus() } + .buttonStyle(.bordered) + + Text("Gateway auto-starts in local mode via launchd (\(gatewayLaunchdLabel)).") + .font(.caption) + .foregroundStyle(.secondary) + .lineLimit(2) + } + .padding(12) + .background(Color.gray.opacity(0.08)) + .cornerRadius(10) + } + + private func refreshGatewayStatus() { + Task { + let status = await Task.detached(priority: .utility) { + GatewayEnvironment.check() + }.value + self.gatewayStatus = status + } + } + + private var gatewayStatusColor: Color { + switch self.gatewayStatus.kind { + case .ok: .green + case .checking: .secondary + case .missingNode, .missingGateway, .incompatible, .error: .orange + } + } + + private var healthCard: some View { + let snapshot = self.healthStore.snapshot + return VStack(alignment: .leading, spacing: 6) { + HStack(spacing: 8) { + Circle() + .fill(self.healthStore.state.tint) + .frame(width: 10, height: 10) + Text(self.healthStore.summaryLine) + .font(.callout.weight(.semibold)) + } + + if let snap = snapshot { + let linkId = snap.channelOrder?.first(where: { + if let summary = snap.channels[$0] { return summary.linked != nil } + return false + }) ?? snap.channels.keys.first(where: { + if let summary = snap.channels[$0] { return summary.linked != nil } + return false + }) + let linkLabel = + linkId.flatMap { snap.channelLabels?[$0] } ?? + linkId?.capitalized ?? + "Link channel" + let linkAge = linkId.flatMap { snap.channels[$0]?.authAgeMs } + Text("\(linkLabel) auth age: \(healthAgeString(linkAge))") + .font(.caption) + .foregroundStyle(.secondary) + Text("Session store: \(snap.sessions.path) (\(snap.sessions.count) entries)") + .font(.caption) + .foregroundStyle(.secondary) + if let recent = snap.sessions.recent.first { + let lastActivity = recent.updatedAt != nil + ? relativeAge(from: Date(timeIntervalSince1970: (recent.updatedAt ?? 0) / 1000)) + : "unknown" + Text("Last activity: \(recent.key) \(lastActivity)") + .font(.caption) + .foregroundStyle(.secondary) + } + Text("Last check: \(relativeAge(from: self.healthStore.lastSuccess))") + .font(.caption) + .foregroundStyle(.secondary) + } else if let error = self.healthStore.lastError { + Text(error) + .font(.caption) + .foregroundStyle(.red) + } else { + Text("Health check pending…") + .font(.caption) + .foregroundStyle(.secondary) + } + + HStack(spacing: 12) { + Button { + Task { await self.healthStore.refresh(onDemand: true) } + } label: { + if self.healthStore.isRefreshing { + ProgressView().controlSize(.small) + } else { + Label("Run Health Check", systemImage: "arrow.clockwise") + } + } + .disabled(self.healthStore.isRefreshing) + + Divider().frame(height: 18) + + Button { + self.revealLogs() + } label: { + Label("Reveal Logs", systemImage: "doc.text.magnifyingglass") + } + } + } + .padding(12) + .background(Color.gray.opacity(0.08)) + .cornerRadius(10) + } +} + +private enum RemoteStatus: Equatable { + case idle + case checking + case ok + case failed(String) +} + +extension GeneralSettings { + private var healthRow: some View { + VStack(alignment: .leading, spacing: 6) { + HStack(spacing: 10) { + Circle() + .fill(self.healthStore.state.tint) + .frame(width: 10, height: 10) + Text(self.healthStore.summaryLine) + .font(.callout) + .frame(maxWidth: .infinity, alignment: .leading) + } + + if let detail = self.healthStore.detailLine { + Text(detail) + .font(.caption) + .foregroundStyle(.secondary) + .fixedSize(horizontal: false, vertical: true) + } + + HStack(spacing: 10) { + Button("Retry now") { + Task { await HealthStore.shared.refresh(onDemand: true) } + } + .disabled(self.healthStore.isRefreshing) + + Button("Open logs") { self.revealLogs() } + .buttonStyle(.link) + .foregroundStyle(.secondary) + } + .font(.caption) + } + } + + @MainActor + func testRemote() async { + self.remoteStatus = .checking + let settings = CommandResolver.connectionSettings() + if self.state.remoteTransport == .direct { + let trimmedUrl = self.state.remoteUrl.trimmingCharacters(in: .whitespacesAndNewlines) + guard !trimmedUrl.isEmpty else { + self.remoteStatus = .failed("Set a gateway URL first") + return + } + guard Self.isValidWsUrl(trimmedUrl) else { + self.remoteStatus = .failed("Gateway URL must start with ws:// or wss://") + return + } + } else { + guard !settings.target.isEmpty else { + self.remoteStatus = .failed("Set an SSH target first") + return + } + + // Step 1: basic SSH reachability check + guard let sshCommand = Self.sshCheckCommand( + target: settings.target, + identity: settings.identity) + else { + self.remoteStatus = .failed("SSH target is invalid") + return + } + let sshResult = await ShellExecutor.run( + command: sshCommand, + cwd: nil, + env: nil, + timeout: 8) + + guard sshResult.ok else { + self.remoteStatus = .failed(self.formatSSHFailure(sshResult, target: settings.target)) + return + } + } + + // Step 2: control channel health check + let originalMode = AppStateStore.shared.connectionMode + do { + try await ControlChannel.shared.configure(mode: .remote( + target: settings.target, + identity: settings.identity)) + let data = try await ControlChannel.shared.health(timeout: 10) + if decodeHealthSnapshot(from: data) != nil { + self.remoteStatus = .ok + } else { + self.remoteStatus = .failed("Control channel returned invalid health JSON") + } + } catch { + self.remoteStatus = .failed(error.localizedDescription) + } + + // Restore original mode if we temporarily switched + switch originalMode { + case .remote: + break + case .local: + try? await ControlChannel.shared.configure(mode: .local) + case .unconfigured: + await ControlChannel.shared.disconnect() + } + } + + private static func isValidWsUrl(_ raw: String) -> Bool { + guard let url = URL(string: raw.trimmingCharacters(in: .whitespacesAndNewlines)) else { return false } + let scheme = url.scheme?.lowercased() ?? "" + guard scheme == "ws" || scheme == "wss" else { return false } + let host = url.host?.trimmingCharacters(in: .whitespacesAndNewlines) ?? "" + return !host.isEmpty + } + + private static func sshCheckCommand(target: String, identity: String) -> [String]? { + guard let parsed = CommandResolver.parseSSHTarget(target) else { return nil } + let options = [ + "-o", "BatchMode=yes", + "-o", "ConnectTimeout=5", + "-o", "StrictHostKeyChecking=accept-new", + "-o", "UpdateHostKeys=yes", + ] + let args = CommandResolver.sshArguments( + target: parsed, + identity: identity, + options: options, + remoteCommand: ["echo", "ok"]) + return ["/usr/bin/ssh"] + args + } + + private func formatSSHFailure(_ response: Response, target: String) -> String { + let payload = response.payload.flatMap { String(data: $0, encoding: .utf8) } + let trimmed = payload? + .trimmingCharacters(in: .whitespacesAndNewlines) + .split(whereSeparator: \.isNewline) + .joined(separator: " ") + if let trimmed, + trimmed.localizedCaseInsensitiveContains("host key verification failed") + { + let host = CommandResolver.parseSSHTarget(target)?.host ?? target + return "SSH check failed: Host key verification failed. Remove the old key with " + + "`ssh-keygen -R \(host)` and try again." + } + if let trimmed, !trimmed.isEmpty { + if let message = response.message, message.hasPrefix("exit ") { + return "SSH check failed: \(trimmed) (\(message))" + } + return "SSH check failed: \(trimmed)" + } + if let message = response.message { + return "SSH check failed (\(message))" + } + return "SSH check failed" + } + + private func revealLogs() { + let target = LogLocator.bestLogFile() + + if let target { + NSWorkspace.shared.selectFile( + target.path, + inFileViewerRootedAtPath: target.deletingLastPathComponent().path) + return + } + + let alert = NSAlert() + alert.messageText = "Log file not found" + alert.informativeText = """ + Looked for openclaw logs in /tmp/openclaw/. + Run a health check or send a message to generate activity, then try again. + """ + alert.alertStyle = .informational + alert.addButton(withTitle: "OK") + alert.runModal() + } + + private func applyDiscoveredGateway(_ gateway: GatewayDiscoveryModel.DiscoveredGateway) { + MacNodeModeCoordinator.shared.setPreferredGatewayStableID(gateway.stableID) + + let host = gateway.tailnetDns ?? gateway.lanHost + guard let host else { return } + let user = NSUserName() + if self.state.remoteTransport == .direct { + if let url = GatewayDiscoveryHelpers.directUrl(for: gateway) { + self.state.remoteUrl = url + } + } else { + self.state.remoteTarget = GatewayDiscoveryModel.buildSSHTarget( + user: user, + host: host, + port: gateway.sshPort) + self.state.remoteCliPath = gateway.cliPath ?? "" + OpenClawConfigFile.setRemoteGatewayUrl(host: host, port: gateway.gatewayPort) + } + } +} + +private func healthAgeString(_ ms: Double?) -> String { + guard let ms else { return "unknown" } + return msToAge(ms) +} + +#if DEBUG +struct GeneralSettings_Previews: PreviewProvider { + static var previews: some View { + GeneralSettings(state: .preview) + .frame(width: SettingsTab.windowWidth, height: SettingsTab.windowHeight) + .environment(TailscaleService.shared) + } +} + +@MainActor +extension GeneralSettings { + static func exerciseForTesting() { + let state = AppState(preview: true) + state.connectionMode = .remote + state.remoteTransport = .ssh + state.remoteTarget = "user@host:2222" + state.remoteUrl = "wss://gateway.example.ts.net" + state.remoteIdentity = "/tmp/id_ed25519" + state.remoteProjectRoot = "/tmp/openclaw" + state.remoteCliPath = "/tmp/openclaw" + + let view = GeneralSettings(state: state) + view.gatewayStatus = GatewayEnvironmentStatus( + kind: .ok, + nodeVersion: "1.0.0", + gatewayVersion: "1.0.0", + requiredGateway: nil, + message: "Gateway ready") + view.remoteStatus = .failed("SSH failed") + view.showRemoteAdvanced = true + _ = view.body + + state.connectionMode = .unconfigured + _ = view.body + + state.connectionMode = .local + view.gatewayStatus = GatewayEnvironmentStatus( + kind: .error("Gateway offline"), + nodeVersion: nil, + gatewayVersion: nil, + requiredGateway: nil, + message: "Gateway offline") + _ = view.body + } +} +#endif diff --git a/apps/macos/Sources/OpenClaw/HealthStore.swift b/apps/macos/Sources/OpenClaw/HealthStore.swift new file mode 100644 index 0000000000000000000000000000000000000000..4fb08f0c3da79754f131f64bef977da01196ff07 --- /dev/null +++ b/apps/macos/Sources/OpenClaw/HealthStore.swift @@ -0,0 +1,301 @@ +import Foundation +import Network +import Observation +import SwiftUI + +struct HealthSnapshot: Codable, Sendable { + struct ChannelSummary: Codable, Sendable { + struct Probe: Codable, Sendable { + struct Bot: Codable, Sendable { + let username: String? + } + + struct Webhook: Codable, Sendable { + let url: String? + } + + let ok: Bool? + let status: Int? + let error: String? + let elapsedMs: Double? + let bot: Bot? + let webhook: Webhook? + } + + let configured: Bool? + let linked: Bool? + let authAgeMs: Double? + let probe: Probe? + let lastProbeAt: Double? + } + + struct SessionInfo: Codable, Sendable { + let key: String + let updatedAt: Double? + let age: Double? + } + + struct Sessions: Codable, Sendable { + let path: String + let count: Int + let recent: [SessionInfo] + } + + let ok: Bool? + let ts: Double + let durationMs: Double + let channels: [String: ChannelSummary] + let channelOrder: [String]? + let channelLabels: [String: String]? + let heartbeatSeconds: Int? + let sessions: Sessions +} + +enum HealthState: Equatable { + case unknown + case ok + case linkingNeeded + case degraded(String) + + var tint: Color { + switch self { + case .ok: .green + case .linkingNeeded: .red + case .degraded: .orange + case .unknown: .secondary + } + } +} + +@MainActor +@Observable +final class HealthStore { + static let shared = HealthStore() + + private static let logger = Logger(subsystem: "ai.openclaw", category: "health") + + private(set) var snapshot: HealthSnapshot? + private(set) var lastSuccess: Date? + private(set) var lastError: String? + private(set) var isRefreshing = false + + private var loopTask: Task? + private let refreshInterval: TimeInterval = 60 + + private init() { + // Avoid background health polling in SwiftUI previews and tests. + if !ProcessInfo.processInfo.isPreview, !ProcessInfo.processInfo.isRunningTests { + self.start() + } + } + + // Test-only escape hatch: the HealthStore is a process-wide singleton but + // state derivation is pure from `snapshot` + `lastError`. + func __setSnapshotForTest(_ snapshot: HealthSnapshot?, lastError: String? = nil) { + self.snapshot = snapshot + self.lastError = lastError + } + + func start() { + guard self.loopTask == nil else { return } + self.loopTask = Task { [weak self] in + guard let self else { return } + while !Task.isCancelled { + await self.refresh() + try? await Task.sleep(nanoseconds: UInt64(self.refreshInterval * 1_000_000_000)) + } + } + } + + func stop() { + self.loopTask?.cancel() + self.loopTask = nil + } + + func refresh(onDemand: Bool = false) async { + guard !self.isRefreshing else { return } + self.isRefreshing = true + defer { self.isRefreshing = false } + let previousError = self.lastError + + do { + let data = try await ControlChannel.shared.health(timeout: 15) + if let decoded = decodeHealthSnapshot(from: data) { + self.snapshot = decoded + self.lastSuccess = Date() + self.lastError = nil + if previousError != nil { + Self.logger.info("health refresh recovered") + } + } else { + self.lastError = "health output not JSON" + if onDemand { self.snapshot = nil } + if previousError != self.lastError { + Self.logger.warning("health refresh failed: output not JSON") + } + } + } catch { + let desc = error.localizedDescription + self.lastError = desc + if onDemand { self.snapshot = nil } + if previousError != desc { + Self.logger.error("health refresh failed \(desc, privacy: .public)") + } + } + } + + private static func isChannelHealthy(_ summary: HealthSnapshot.ChannelSummary) -> Bool { + guard summary.configured == true else { return false } + // If probe is missing, treat it as "configured but unknown health" (not a hard fail). + return summary.probe?.ok ?? true + } + + private static func describeProbeFailure(_ probe: HealthSnapshot.ChannelSummary.Probe) -> String { + let elapsed = probe.elapsedMs.map { "\(Int($0))ms" } + if let error = probe.error, error.lowercased().contains("timeout") || probe.status == nil { + if let elapsed { return "Health check timed out (\(elapsed))" } + return "Health check timed out" + } + let code = probe.status.map { "status \($0)" } ?? "status unknown" + let reason = probe.error?.isEmpty == false ? probe.error! : "health probe failed" + if let elapsed { return "\(reason) (\(code), \(elapsed))" } + return "\(reason) (\(code))" + } + + private func resolveLinkChannel( + _ snap: HealthSnapshot) -> (id: String, summary: HealthSnapshot.ChannelSummary)? + { + let order = snap.channelOrder ?? Array(snap.channels.keys) + for id in order { + if let summary = snap.channels[id], summary.linked == true { + return (id: id, summary: summary) + } + } + for id in order { + if let summary = snap.channels[id], summary.linked != nil { + return (id: id, summary: summary) + } + } + return nil + } + + private func resolveFallbackChannel( + _ snap: HealthSnapshot, + excluding id: String?) -> (id: String, summary: HealthSnapshot.ChannelSummary)? + { + let order = snap.channelOrder ?? Array(snap.channels.keys) + for channelId in order { + if channelId == id { continue } + guard let summary = snap.channels[channelId] else { continue } + if Self.isChannelHealthy(summary) { + return (id: channelId, summary: summary) + } + } + return nil + } + + var state: HealthState { + if let error = self.lastError, !error.isEmpty { + return .degraded(error) + } + guard let snap = self.snapshot else { return .unknown } + guard let link = self.resolveLinkChannel(snap) else { return .unknown } + if link.summary.linked != true { + // Linking is optional if any other channel is healthy; don't paint the whole app red. + let fallback = self.resolveFallbackChannel(snap, excluding: link.id) + return fallback != nil ? .degraded("Not linked") : .linkingNeeded + } + // A channel can be "linked" but still unhealthy (failed probe / cannot connect). + if let probe = link.summary.probe, probe.ok == false { + return .degraded(Self.describeProbeFailure(probe)) + } + return .ok + } + + var summaryLine: String { + if self.isRefreshing { return "Health check running…" } + if let error = self.lastError { return "Health check failed: \(error)" } + guard let snap = self.snapshot else { return "Health check pending" } + guard let link = self.resolveLinkChannel(snap) else { return "Health check pending" } + if link.summary.linked != true { + if let fallback = self.resolveFallbackChannel(snap, excluding: link.id) { + let fallbackLabel = snap.channelLabels?[fallback.id] ?? fallback.id.capitalized + let fallbackState = (fallback.summary.probe?.ok ?? true) ? "ok" : "degraded" + return "\(fallbackLabel) \(fallbackState) · Not linked — run openclaw login" + } + return "Not linked — run openclaw login" + } + let auth = link.summary.authAgeMs.map { msToAge($0) } ?? "unknown" + if let probe = link.summary.probe, probe.ok == false { + let status = probe.status.map(String.init) ?? "?" + let suffix = probe.status == nil ? "probe degraded" : "probe degraded · status \(status)" + return "linked · auth \(auth) · \(suffix)" + } + return "linked · auth \(auth)" + } + + /// Short, human-friendly detail for the last failure, used in the UI. + var detailLine: String? { + if let error = self.lastError, !error.isEmpty { + let lower = error.lowercased() + if lower.contains("connection refused") { + let port = GatewayEnvironment.gatewayPort() + let host = GatewayConnectivityCoordinator.shared.localEndpointHostLabel ?? "127.0.0.1:\(port)" + return "The gateway control port (\(host)) isn’t listening — restart OpenClaw to bring it back." + } + if lower.contains("timeout") { + return "Timed out waiting for the control server; the gateway may be crashed or still starting." + } + return error + } + return nil + } + + func describeFailure(from snap: HealthSnapshot, fallback: String?) -> String { + if let link = self.resolveLinkChannel(snap), link.summary.linked != true { + return "Not linked — run openclaw login" + } + if let link = self.resolveLinkChannel(snap), let probe = link.summary.probe, probe.ok == false { + return Self.describeProbeFailure(probe) + } + if let fallback, !fallback.isEmpty { + return fallback + } + return "health probe failed" + } + + var degradedSummary: String? { + guard case let .degraded(reason) = self.state else { return nil } + if reason == "[object Object]" || reason.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty, + let snap = self.snapshot + { + return self.describeFailure(from: snap, fallback: reason) + } + return reason + } +} + +func msToAge(_ ms: Double) -> String { + let minutes = Int(round(ms / 60000)) + if minutes < 1 { return "just now" } + if minutes < 60 { return "\(minutes)m" } + let hours = Int(round(Double(minutes) / 60)) + if hours < 48 { return "\(hours)h" } + let days = Int(round(Double(hours) / 24)) + return "\(days)d" +} + +/// Decode a health snapshot, tolerating stray log lines before/after the JSON blob. +func decodeHealthSnapshot(from data: Data) -> HealthSnapshot? { + let decoder = JSONDecoder() + if let snap = try? decoder.decode(HealthSnapshot.self, from: data) { + return snap + } + guard let text = String(data: data, encoding: .utf8) else { return nil } + guard let firstBrace = text.firstIndex(of: "{"), let lastBrace = text.lastIndex(of: "}") else { + return nil + } + let slice = text[firstBrace...lastBrace] + let cleaned = Data(slice.utf8) + return try? decoder.decode(HealthSnapshot.self, from: cleaned) +} diff --git a/apps/macos/Sources/OpenClaw/HeartbeatStore.swift b/apps/macos/Sources/OpenClaw/HeartbeatStore.swift new file mode 100644 index 0000000000000000000000000000000000000000..6bd7bb52529d594bd1756f201aa4e9495bbd1cfa --- /dev/null +++ b/apps/macos/Sources/OpenClaw/HeartbeatStore.swift @@ -0,0 +1,39 @@ +import Foundation +import Observation +import SwiftUI + +@MainActor +@Observable +final class HeartbeatStore { + static let shared = HeartbeatStore() + + private(set) var lastEvent: ControlHeartbeatEvent? + + private var observer: NSObjectProtocol? + + private init() { + self.observer = NotificationCenter.default.addObserver( + forName: .controlHeartbeat, + object: nil, + queue: .main) + { [weak self] note in + guard let data = note.object as? Data else { return } + if let decoded = try? JSONDecoder().decode(ControlHeartbeatEvent.self, from: data) { + Task { @MainActor in self?.lastEvent = decoded } + } + } + + Task { + if self.lastEvent == nil { + if let evt = try? await ControlChannel.shared.lastHeartbeat() { + self.lastEvent = evt + } + } + } + } + + @MainActor + deinit { + if let observer { NotificationCenter.default.removeObserver(observer) } + } +} diff --git a/apps/macos/Sources/OpenClaw/HoverHUD.swift b/apps/macos/Sources/OpenClaw/HoverHUD.swift new file mode 100644 index 0000000000000000000000000000000000000000..d3482362a0f4b17698caf8b717694513a546a528 --- /dev/null +++ b/apps/macos/Sources/OpenClaw/HoverHUD.swift @@ -0,0 +1,311 @@ +import AppKit +import Observation +import QuartzCore +import SwiftUI + +/// Hover-only HUD anchored to the menu bar item. Click expands into full Web Chat. +@MainActor +@Observable +final class HoverHUDController { + static let shared = HoverHUDController() + + struct Model { + var isVisible: Bool = false + var isSuppressed: Bool = false + var hoveringStatusItem: Bool = false + var hoveringPanel: Bool = false + } + + private(set) var model = Model() + + private var window: NSPanel? + private var hostingView: NSHostingView? + private var dismissMonitor: Any? + private var dismissTask: Task? + private var showTask: Task? + private var anchorProvider: (() -> NSRect?)? + + private let width: CGFloat = 360 + private let height: CGFloat = 74 + private let padding: CGFloat = 8 + private let hoverShowDelay: TimeInterval = 0.18 + + func setSuppressed(_ suppressed: Bool) { + self.model.isSuppressed = suppressed + if suppressed { + self.showTask?.cancel() + self.showTask = nil + self.dismiss(reason: "suppressed") + } + } + + func statusItemHoverChanged(inside: Bool, anchorProvider: @escaping () -> NSRect?) { + self.model.hoveringStatusItem = inside + self.anchorProvider = anchorProvider + + guard !self.model.isSuppressed else { return } + + if inside { + self.dismissTask?.cancel() + self.dismissTask = nil + self.showTask?.cancel() + self.showTask = Task { [weak self] in + guard let self else { return } + try? await Task.sleep(nanoseconds: UInt64(self.hoverShowDelay * 1_000_000_000)) + await MainActor.run { [weak self] in + guard let self else { return } + guard !Task.isCancelled else { return } + guard self.model.hoveringStatusItem else { return } + guard !self.model.isSuppressed else { return } + self.present() + } + } + } else { + self.showTask?.cancel() + self.showTask = nil + self.scheduleDismiss() + } + } + + func panelHoverChanged(inside: Bool) { + self.model.hoveringPanel = inside + if inside { + self.dismissTask?.cancel() + self.dismissTask = nil + } else if !self.model.hoveringStatusItem { + self.scheduleDismiss() + } + } + + func openChat() { + guard let anchorProvider = self.anchorProvider else { return } + self.dismiss(reason: "openChat") + Task { @MainActor in + let sessionKey = await WebChatManager.shared.preferredSessionKey() + WebChatManager.shared.togglePanel(sessionKey: sessionKey, anchorProvider: anchorProvider) + } + } + + func dismiss(reason: String = "explicit") { + self.dismissTask?.cancel() + self.dismissTask = nil + self.removeDismissMonitor() + guard let window else { + self.model.isVisible = false + return + } + + if !self.model.isVisible { + window.orderOut(nil) + return + } + + let target = window.frame.offsetBy(dx: 0, dy: 6) + NSAnimationContext.runAnimationGroup { context in + context.duration = 0.14 + context.timingFunction = CAMediaTimingFunction(name: .easeOut) + window.animator().setFrame(target, display: true) + window.animator().alphaValue = 0 + } completionHandler: { + Task { @MainActor in + window.orderOut(nil) + self.model.isVisible = false + } + } + } + + // MARK: - Private + + private func scheduleDismiss() { + self.dismissTask?.cancel() + self.dismissTask = Task { [weak self] in + try? await Task.sleep(nanoseconds: 250_000_000) + await MainActor.run { + guard let self else { return } + if self.model.hoveringStatusItem || self.model.hoveringPanel { return } + self.dismiss(reason: "hoverExit") + } + } + } + + private func present() { + guard !self.model.isSuppressed else { return } + self.ensureWindow() + self.hostingView?.rootView = HoverHUDView(controller: self) + let target = self.targetFrame() + + guard let window else { return } + self.installDismissMonitor() + + if !self.model.isVisible { + self.model.isVisible = true + let start = target.offsetBy(dx: 0, dy: 8) + window.setFrame(start, display: true) + window.alphaValue = 0 + window.orderFrontRegardless() + NSAnimationContext.runAnimationGroup { context in + context.duration = 0.18 + context.timingFunction = CAMediaTimingFunction(name: .easeOut) + window.animator().setFrame(target, display: true) + window.animator().alphaValue = 1 + } + } else { + window.orderFrontRegardless() + self.updateWindowFrame(animate: true) + } + } + + private func ensureWindow() { + if self.window != nil { return } + let panel = NSPanel( + contentRect: NSRect(x: 0, y: 0, width: self.width, height: self.height), + styleMask: [.nonactivatingPanel, .borderless], + backing: .buffered, + defer: false) + panel.isOpaque = false + panel.backgroundColor = .clear + panel.hasShadow = true + panel.level = .statusBar + panel.collectionBehavior = [.canJoinAllSpaces, .fullScreenAuxiliary, .transient] + panel.hidesOnDeactivate = false + panel.isMovable = false + panel.isFloatingPanel = true + panel.becomesKeyOnlyIfNeeded = true + panel.titleVisibility = .hidden + panel.titlebarAppearsTransparent = true + + let host = NSHostingView(rootView: HoverHUDView(controller: self)) + host.translatesAutoresizingMaskIntoConstraints = false + panel.contentView = host + self.hostingView = host + self.window = panel + } + + private func targetFrame() -> NSRect { + guard let anchor = self.anchorProvider?() else { + return WindowPlacement.topRightFrame( + size: NSSize(width: self.width, height: self.height), + padding: self.padding) + } + + let screen = NSScreen.screens.first { screen in + screen.frame.contains(anchor.origin) || screen.frame.contains(NSPoint(x: anchor.midX, y: anchor.midY)) + } ?? NSScreen.main + + let bounds = (screen?.visibleFrame ?? .zero).insetBy(dx: self.padding, dy: self.padding) + return WindowPlacement.anchoredBelowFrame( + size: NSSize(width: self.width, height: self.height), + anchor: anchor, + padding: self.padding, + in: bounds) + } + + private func updateWindowFrame(animate: Bool = false) { + guard let window else { return } + let frame = self.targetFrame() + if animate { + NSAnimationContext.runAnimationGroup { context in + context.duration = 0.12 + context.timingFunction = CAMediaTimingFunction(name: .easeOut) + window.animator().setFrame(frame, display: true) + } + } else { + window.setFrame(frame, display: true) + } + } + + private func installDismissMonitor() { + if ProcessInfo.processInfo.isRunningTests { return } + guard self.dismissMonitor == nil, let window else { return } + self.dismissMonitor = NSEvent.addGlobalMonitorForEvents(matching: [ + .leftMouseDown, + .rightMouseDown, + .otherMouseDown, + ]) { [weak self] _ in + guard let self, self.model.isVisible else { return } + let pt = NSEvent.mouseLocation + if !window.frame.contains(pt) { + Task { @MainActor in self.dismiss(reason: "outsideClick") } + } + } + } + + private func removeDismissMonitor() { + if let monitor = self.dismissMonitor { + NSEvent.removeMonitor(monitor) + self.dismissMonitor = nil + } + } +} + +private struct HoverHUDView: View { + var controller: HoverHUDController + private let activityStore = WorkActivityStore.shared + + private var statusTitle: String { + if self.activityStore.iconState.isWorking { return "Working" } + return "Idle" + } + + private var detail: String { + if let current = self.activityStore.current?.label, !current.isEmpty { return current } + if let last = self.activityStore.lastToolLabel, !last.isEmpty { return last } + return "No recent activity" + } + + private var symbolName: String { + if self.activityStore.iconState.isWorking { + return self.activityStore.iconState.badgeSymbolName + } + return "moon.zzz.fill" + } + + private var dotColor: Color { + if self.activityStore.iconState.isWorking { + return Color(nsColor: NSColor.systemGreen.withAlphaComponent(0.7)) + } + return .secondary + } + + var body: some View { + HStack(alignment: .top, spacing: 10) { + Circle() + .fill(self.dotColor) + .frame(width: 7, height: 7) + .padding(.top, 5) + + VStack(alignment: .leading, spacing: 4) { + Text(self.statusTitle) + .font(.system(size: 13, weight: .semibold)) + .foregroundStyle(.primary) + Text(self.detail) + .font(.system(size: 12)) + .foregroundStyle(.secondary) + .lineLimit(2) + .truncationMode(.middle) + .fixedSize(horizontal: false, vertical: true) + } + + Spacer(minLength: 8) + + Image(systemName: self.symbolName) + .font(.system(size: 14, weight: .semibold)) + .foregroundStyle(.secondary) + .padding(.top, 1) + } + .padding(12) + .background( + RoundedRectangle(cornerRadius: 14, style: .continuous) + .fill(.regularMaterial)) + .overlay( + RoundedRectangle(cornerRadius: 14, style: .continuous) + .strokeBorder(Color.black.opacity(0.10), lineWidth: 1)) + .contentShape(Rectangle()) + .onHover { inside in + self.controller.panelHoverChanged(inside: inside) + } + .onTapGesture { + self.controller.openChat() + } + } +} diff --git a/apps/macos/Sources/OpenClaw/IconState.swift b/apps/macos/Sources/OpenClaw/IconState.swift new file mode 100644 index 0000000000000000000000000000000000000000..ec27385835428d5524f61d546486eac36d7b1772 --- /dev/null +++ b/apps/macos/Sources/OpenClaw/IconState.swift @@ -0,0 +1,111 @@ +import Foundation +import SwiftUI + +enum SessionRole { + case main + case other +} + +enum ToolKind: String, Codable { + case bash, read, write, edit, attach, other +} + +enum ActivityKind: Codable, Equatable { + case job + case tool(ToolKind) +} + +enum IconState: Equatable { + case idle + case workingMain(ActivityKind) + case workingOther(ActivityKind) + case overridden(ActivityKind) + + enum BadgeProminence: Equatable { + case primary + case secondary + case overridden + } + + var badgeSymbolName: String { + switch self.activity { + case .tool(.bash): "chevron.left.slash.chevron.right" + case .tool(.read): "doc" + case .tool(.write): "pencil" + case .tool(.edit): "pencil.tip" + case .tool(.attach): "paperclip" + case .tool(.other), .job: "gearshape.fill" + } + } + + var badgeProminence: BadgeProminence? { + switch self { + case .idle: nil + case .workingMain: .primary + case .workingOther: .secondary + case .overridden: .overridden + } + } + + var isWorking: Bool { + switch self { + case .idle: false + default: true + } + } + + private var activity: ActivityKind { + switch self { + case let .workingMain(kind), + let .workingOther(kind), + let .overridden(kind): + kind + case .idle: + .job + } + } +} + +enum IconOverrideSelection: String, CaseIterable, Identifiable { + case system + case idle + case mainBash, mainRead, mainWrite, mainEdit, mainOther + case otherBash, otherRead, otherWrite, otherEdit, otherOther + + var id: String { self.rawValue } + + var label: String { + switch self { + case .system: "System (auto)" + case .idle: "Idle" + case .mainBash: "Working main – bash" + case .mainRead: "Working main – read" + case .mainWrite: "Working main – write" + case .mainEdit: "Working main – edit" + case .mainOther: "Working main – other" + case .otherBash: "Working other – bash" + case .otherRead: "Working other – read" + case .otherWrite: "Working other – write" + case .otherEdit: "Working other – edit" + case .otherOther: "Working other – other" + } + } + + func toIconState() -> IconState { + let map: (ToolKind) -> ActivityKind = { .tool($0) } + switch self { + case .system: return .idle + case .idle: return .idle + case .mainBash: return .workingMain(map(.bash)) + case .mainRead: return .workingMain(map(.read)) + case .mainWrite: return .workingMain(map(.write)) + case .mainEdit: return .workingMain(map(.edit)) + case .mainOther: return .workingMain(map(.other)) + case .otherBash: return .workingOther(map(.bash)) + case .otherRead: return .workingOther(map(.read)) + case .otherWrite: return .workingOther(map(.write)) + case .otherEdit: return .workingOther(map(.edit)) + case .otherOther: return .workingOther(map(.other)) + } + } +} diff --git a/apps/macos/Sources/OpenClaw/InstancesSettings.swift b/apps/macos/Sources/OpenClaw/InstancesSettings.swift new file mode 100644 index 0000000000000000000000000000000000000000..0c992c6970fab5bce1536a4af7f2efdbc1af02a5 --- /dev/null +++ b/apps/macos/Sources/OpenClaw/InstancesSettings.swift @@ -0,0 +1,479 @@ +import AppKit +import SwiftUI + +struct InstancesSettings: View { + var store: InstancesStore + + init(store: InstancesStore = .shared) { + self.store = store + } + + var body: some View { + VStack(alignment: .leading, spacing: 12) { + self.header + if let err = store.lastError { + Text("Error: \(err)") + .foregroundStyle(.red) + } else if let info = store.statusMessage { + Text(info) + .foregroundStyle(.secondary) + } + if self.store.instances.isEmpty { + Text("No instances reported yet.") + .foregroundStyle(.secondary) + } else { + List(self.store.instances) { inst in + self.instanceRow(inst) + } + .listStyle(.inset) + } + Spacer() + } + .onAppear { self.store.start() } + .onDisappear { self.store.stop() } + } + + private var header: some View { + HStack { + VStack(alignment: .leading, spacing: 4) { + Text("Connected Instances") + .font(.headline) + Text("Latest presence beacons from OpenClaw nodes. Updated periodically.") + .font(.footnote) + .foregroundStyle(.secondary) + } + Spacer() + if self.store.isLoading { + ProgressView() + } else { + Button { + Task { await self.store.refresh() } + } label: { + Label("Refresh", systemImage: "arrow.clockwise") + } + .buttonStyle(.bordered) + .help("Refresh") + } + } + } + + @ViewBuilder + private func instanceRow(_ inst: InstanceInfo) -> some View { + let isGateway = (inst.mode ?? "").trimmingCharacters(in: .whitespacesAndNewlines).lowercased() == "gateway" + let prettyPlatform = inst.platform.flatMap { self.prettyPlatform($0) } + let device = DeviceModelCatalog.presentation( + deviceFamily: inst.deviceFamily, + modelIdentifier: inst.modelIdentifier) + + HStack(alignment: .top, spacing: 12) { + self.leadingDeviceIcon(inst, device: device) + .frame(width: 28, height: 28, alignment: .center) + .padding(.top, 1) + + VStack(alignment: .leading, spacing: 4) { + HStack(spacing: 8) { + Text(inst.host ?? "unknown host").font(.subheadline.bold()) + self.presenceIndicator(inst) + if let ip = inst.ip { Text("(") + Text(ip).monospaced() + Text(")") } + } + + HStack(spacing: 8) { + if let version = inst.version { + self.label(icon: "shippingbox", text: version) + } + + if let device { + // Avoid showing generic "Mac"/"iPhone"/etc; prefer the concrete model name. + let family = (inst.deviceFamily ?? "").trimmingCharacters(in: .whitespacesAndNewlines) + let isGeneric = !family.isEmpty && device.title == family + if !isGeneric { + if let prettyPlatform { + self.label(icon: device.symbol, text: "\(device.title) · \(prettyPlatform)") + } else { + self.label(icon: device.symbol, text: device.title) + } + } else if let prettyPlatform, let platform = inst.platform { + self.label(icon: self.platformIcon(platform), text: prettyPlatform) + } + } else if let prettyPlatform, let platform = inst.platform { + self.label(icon: self.platformIcon(platform), text: prettyPlatform) + } + + if let mode = inst.mode { self.label(icon: "network", text: mode) } + } + .layoutPriority(1) + + if !isGateway, self.shouldShowUpdateRow(inst) { + HStack(spacing: 8) { + Spacer(minLength: 0) + + // Last local input is helpful for interactive nodes, but noisy/meaningless for the gateway. + if let secs = inst.lastInputSeconds { + self.label(icon: "clock", text: "\(secs)s ago") + } + + if let update = self.updateSummaryText(inst, isGateway: isGateway) { + self.label(icon: "arrow.clockwise", text: update) + .help(self.presenceUpdateSourceHelp(inst.reason ?? "")) + } + } + .foregroundStyle(.secondary) + } + } + } + .padding(.vertical, 6) + .help(inst.text) + .contextMenu { + Button("Copy Debug Summary") { + NSPasteboard.general.clearContents() + NSPasteboard.general.setString(inst.text, forType: .string) + } + } + } + + private func label(icon: String?, text: String) -> some View { + HStack(spacing: 4) { + if let icon { + if icon == Self.androidSymbolToken { + AndroidMark() + .foregroundStyle(.secondary) + .frame(width: 12, height: 12, alignment: .center) + } else if self.isSystemSymbolAvailable(icon) { + Image(systemName: icon).foregroundStyle(.secondary).font(.caption) + } + } + Text(text) + } + .font(.footnote) + } + + private func presenceIndicator(_ inst: InstanceInfo) -> some View { + let status = self.presenceStatus(for: inst) + return HStack(spacing: 4) { + Circle() + .fill(status.color) + .frame(width: 6, height: 6) + .accessibilityHidden(true) + Text(status.label) + .foregroundStyle(.secondary) + } + .font(.caption) + .help("Presence updated \(inst.ageDescription).") + .accessibilityLabel("\(status.label) presence") + } + + private func presenceStatus(for inst: InstanceInfo) -> (label: String, color: Color) { + let nowMs = Date().timeIntervalSince1970 * 1000 + let ageSeconds = max(0, Int((nowMs - inst.ts) / 1000)) + if ageSeconds <= 120 { return ("Active", .green) } + if ageSeconds <= 300 { return ("Idle", .yellow) } + return ("Stale", .gray) + } + + @ViewBuilder + private func leadingDeviceIcon(_ inst: InstanceInfo, device: DevicePresentation?) -> some View { + let symbol = self.leadingDeviceSymbol(inst, device: device) + if symbol == Self.androidSymbolToken { + AndroidMark() + .foregroundStyle(.secondary) + .frame(width: 24, height: 24, alignment: .center) + .accessibilityHidden(true) + } else { + Image(systemName: symbol) + .font(.system(size: 26, weight: .regular)) + .foregroundStyle(.secondary) + .accessibilityHidden(true) + } + } + + private static let androidSymbolToken = "android" + + private func leadingDeviceSymbol(_ inst: InstanceInfo, device: DevicePresentation?) -> String { + let family = (inst.deviceFamily ?? "").trimmingCharacters(in: .whitespacesAndNewlines).lowercased() + if family == "android" { + return Self.androidSymbolToken + } + + if let title = device?.title.lowercased() { + if title.contains("mac studio") { + return self.safeSystemSymbol("macstudio", fallback: "desktopcomputer") + } + if title.contains("macbook") { + return self.safeSystemSymbol("laptopcomputer", fallback: "laptopcomputer") + } + if title.contains("ipad") { + return self.safeSystemSymbol("ipad", fallback: "ipad") + } + if title.contains("iphone") { + return self.safeSystemSymbol("iphone", fallback: "iphone") + } + } + + if let symbol = device?.symbol { + return self.safeSystemSymbol(symbol, fallback: "cpu") + } + + if let platform = inst.platform { + return self.safeSystemSymbol(self.platformIcon(platform), fallback: "cpu") + } + + return "cpu" + } + + private func shouldShowUpdateRow(_ inst: InstanceInfo) -> Bool { + if inst.lastInputSeconds != nil { return true } + if self.updateSummaryText(inst, isGateway: false) != nil { return true } + return false + } + + private func safeSystemSymbol(_ preferred: String, fallback: String) -> String { + if self.isSystemSymbolAvailable(preferred) { return preferred } + return fallback + } + + private func isSystemSymbolAvailable(_ name: String) -> Bool { + NSImage(systemSymbolName: name, accessibilityDescription: nil) != nil + } + + private struct AndroidMark: View { + var body: some View { + GeometryReader { geo in + let w = geo.size.width + let h = geo.size.height + let headHeight = h * 0.68 + let headWidth = w * 0.92 + let headY = h * 0.18 + let corner = headHeight * 0.28 + + ZStack { + RoundedRectangle(cornerRadius: corner, style: .continuous) + .frame(width: headWidth, height: headHeight) + .position(x: w / 2, y: headY + headHeight / 2) + + Circle() + .frame(width: max(1, w * 0.1), height: max(1, w * 0.1)) + .position(x: w * 0.38, y: headY + headHeight * 0.55) + .blendMode(.destinationOut) + + Circle() + .frame(width: max(1, w * 0.1), height: max(1, w * 0.1)) + .position(x: w * 0.62, y: headY + headHeight * 0.55) + .blendMode(.destinationOut) + + Rectangle() + .frame(width: max(1, w * 0.08), height: max(1, h * 0.18)) + .rotationEffect(.degrees(-25)) + .position(x: w * 0.34, y: h * 0.12) + + Rectangle() + .frame(width: max(1, w * 0.08), height: max(1, h * 0.18)) + .rotationEffect(.degrees(25)) + .position(x: w * 0.66, y: h * 0.12) + } + .compositingGroup() + } + } + } + + private func platformIcon(_ raw: String) -> String { + let (prefix, _) = self.parsePlatform(raw) + switch prefix { + case "macos": + return "laptopcomputer" + case "ios": + return "iphone" + case "ipados": + return "ipad" + case "tvos": + return "appletv" + case "watchos": + return "applewatch" + default: + return "cpu" + } + } + + private func prettyPlatform(_ raw: String) -> String? { + let (prefix, version) = self.parsePlatform(raw) + if prefix.isEmpty { return nil } + let name: String = switch prefix { + case "macos": "macOS" + case "ios": "iOS" + case "ipados": "iPadOS" + case "tvos": "tvOS" + case "watchos": "watchOS" + default: prefix.prefix(1).uppercased() + prefix.dropFirst() + } + guard let version, !version.isEmpty else { return name } + let parts = version.split(separator: ".").map(String.init) + if parts.count >= 2 { + return "\(name) \(parts[0]).\(parts[1])" + } + return "\(name) \(version)" + } + + private func parsePlatform(_ raw: String) -> (prefix: String, version: String?) { + let trimmed = raw.trimmingCharacters(in: .whitespacesAndNewlines) + if trimmed.isEmpty { return ("", nil) } + let parts = trimmed.split(whereSeparator: { $0 == " " || $0 == "\t" }).map(String.init) + let prefix = parts.first?.lowercased() ?? "" + let versionToken = parts.dropFirst().first + return (prefix, versionToken) + } + + private func presenceUpdateSourceShortText(_ reason: String) -> String? { + let trimmed = reason.trimmingCharacters(in: .whitespacesAndNewlines) + guard !trimmed.isEmpty else { return nil } + switch trimmed { + case "self": + return "Self" + case "connect": + return "Connect" + case "disconnect": + return "Disconnect" + case "node-connected": + return "Node connect" + case "node-disconnected": + return "Node disconnect" + case "launch": + return "Launch" + case "periodic": + return "Heartbeat" + case "instances-refresh": + return "Instances" + case "seq gap": + return "Resync" + default: + return trimmed + } + } + + private func updateSummaryText(_ inst: InstanceInfo, isGateway: Bool) -> String? { + // For gateway rows, omit the "updated via/by" provenance entirely. + if isGateway { + return nil + } + + let age = inst.ageDescription.trimmingCharacters(in: .whitespacesAndNewlines) + guard !age.isEmpty else { return nil } + + let source = self.presenceUpdateSourceShortText(inst.reason ?? "") + if let source, !source.isEmpty { + return "\(age) · \(source)" + } + return age + } + + private func presenceUpdateSourceHelp(_ reason: String) -> String { + let trimmed = reason.trimmingCharacters(in: .whitespacesAndNewlines) + if trimmed.isEmpty { + return "Why this presence entry was last updated (debug marker)." + } + return "Why this presence entry was last updated (debug marker). Raw: \(trimmed)" + } +} + +#if DEBUG +extension InstancesSettings { + static func exerciseForTesting() { + let view = InstancesSettings(store: InstancesStore(isPreview: true)) + let mac = InstanceInfo( + id: "mac", + host: "studio", + ip: "10.0.0.2", + version: "1.2.3", + platform: "macOS 14.2", + deviceFamily: "Mac", + modelIdentifier: "Mac14,10", + lastInputSeconds: 12, + mode: "local", + reason: "self", + text: "Mac Studio", + ts: 1_700_000_000_000) + let genericIOS = InstanceInfo( + id: "iphone", + host: "phone", + ip: "10.0.0.3", + version: "2.0.0", + platform: "iOS 18.0", + deviceFamily: "iPhone", + modelIdentifier: nil, + lastInputSeconds: 35, + mode: "node", + reason: "connect", + text: "iPhone node", + ts: 1_700_000_100_000) + let android = InstanceInfo( + id: "android", + host: "pixel", + ip: nil, + version: "3.1.0", + platform: "Android 14", + deviceFamily: "Android", + modelIdentifier: nil, + lastInputSeconds: 90, + mode: "node", + reason: "seq gap", + text: "Android node", + ts: 1_700_000_200_000) + let gateway = InstanceInfo( + id: "gateway", + host: "gateway", + ip: "10.0.0.9", + version: "4.0.0", + platform: "Linux", + deviceFamily: nil, + modelIdentifier: nil, + lastInputSeconds: nil, + mode: "gateway", + reason: "periodic", + text: "Gateway", + ts: 1_700_000_300_000) + + _ = view.instanceRow(mac) + _ = view.instanceRow(genericIOS) + _ = view.instanceRow(android) + _ = view.instanceRow(gateway) + + _ = view.leadingDeviceSymbol( + mac, + device: DevicePresentation(title: "Mac Studio", symbol: "macstudio")) + _ = view.leadingDeviceSymbol( + mac, + device: DevicePresentation(title: "MacBook Pro", symbol: "laptopcomputer")) + _ = view.leadingDeviceSymbol(android, device: nil) + _ = view.platformIcon("tvOS 17.1") + _ = view.platformIcon("watchOS 10") + _ = view.platformIcon("unknown 1.0") + _ = view.prettyPlatform("macOS 14.2") + _ = view.prettyPlatform("iOS 18") + _ = view.prettyPlatform("ipados 17.1") + _ = view.prettyPlatform("linux") + _ = view.prettyPlatform(" ") + _ = view.parsePlatform("macOS 14.1") + _ = view.parsePlatform(" ") + _ = view.presenceUpdateSourceShortText("self") + _ = view.presenceUpdateSourceShortText("instances-refresh") + _ = view.presenceUpdateSourceShortText("seq gap") + _ = view.presenceUpdateSourceShortText("custom") + _ = view.presenceUpdateSourceShortText(" ") + _ = view.updateSummaryText(mac, isGateway: false) + _ = view.updateSummaryText(gateway, isGateway: true) + _ = view.presenceUpdateSourceHelp("") + _ = view.presenceUpdateSourceHelp("connect") + _ = view.safeSystemSymbol("not-a-symbol", fallback: "cpu") + _ = view.isSystemSymbolAvailable("sparkles") + _ = view.label(icon: "android", text: "Android") + _ = view.label(icon: "sparkles", text: "Sparkles") + _ = view.label(icon: nil, text: "Plain") + _ = AndroidMark().body + } +} + +struct InstancesSettings_Previews: PreviewProvider { + static var previews: some View { + InstancesSettings(store: .preview()) + .frame(width: SettingsTab.windowWidth, height: SettingsTab.windowHeight) + } +} +#endif diff --git a/apps/macos/Sources/OpenClaw/InstancesStore.swift b/apps/macos/Sources/OpenClaw/InstancesStore.swift new file mode 100644 index 0000000000000000000000000000000000000000..1f9dce6cb9a2e6db8e04debe7367119d7f634387 --- /dev/null +++ b/apps/macos/Sources/OpenClaw/InstancesStore.swift @@ -0,0 +1,394 @@ +import OpenClawKit +import OpenClawProtocol +import Cocoa +import Foundation +import Observation +import OSLog + +struct InstanceInfo: Identifiable, Codable { + let id: String + let host: String? + let ip: String? + let version: String? + let platform: String? + let deviceFamily: String? + let modelIdentifier: String? + let lastInputSeconds: Int? + let mode: String? + let reason: String? + let text: String + let ts: Double + + var ageDescription: String { + let date = Date(timeIntervalSince1970: ts / 1000) + return age(from: date) + } + + var lastInputDescription: String { + guard let secs = lastInputSeconds else { return "unknown" } + return "\(secs)s ago" + } +} + +@MainActor +@Observable +final class InstancesStore { + static let shared = InstancesStore() + let isPreview: Bool + + var instances: [InstanceInfo] = [] + var lastError: String? + var statusMessage: String? + var isLoading = false + + private let logger = Logger(subsystem: "ai.openclaw", category: "instances") + private var task: Task? + private let interval: TimeInterval = 30 + private var eventTask: Task? + private var startCount = 0 + private var lastPresenceById: [String: InstanceInfo] = [:] + private var lastLoginNotifiedAtMs: [String: Double] = [:] + + private struct PresenceEventPayload: Codable { + let presence: [PresenceEntry] + } + + init(isPreview: Bool = false) { + self.isPreview = isPreview + } + + func start() { + guard !self.isPreview else { return } + self.startCount += 1 + guard self.startCount == 1 else { return } + guard self.task == nil else { return } + self.startGatewaySubscription() + self.task = Task.detached { [weak self] in + guard let self else { return } + await self.refresh() + while !Task.isCancelled { + try? await Task.sleep(nanoseconds: UInt64(self.interval * 1_000_000_000)) + await self.refresh() + } + } + } + + func stop() { + guard !self.isPreview else { return } + guard self.startCount > 0 else { return } + self.startCount -= 1 + guard self.startCount == 0 else { return } + self.task?.cancel() + self.task = nil + self.eventTask?.cancel() + self.eventTask = nil + } + + private func startGatewaySubscription() { + self.eventTask?.cancel() + self.eventTask = Task { [weak self] in + guard let self else { return } + let stream = await GatewayConnection.shared.subscribe() + for await push in stream { + if Task.isCancelled { return } + await MainActor.run { [weak self] in + self?.handle(push: push) + } + } + } + } + + private func handle(push: GatewayPush) { + switch push { + case let .event(evt) where evt.event == "presence": + if let payload = evt.payload { + self.handlePresenceEventPayload(payload) + } + case .seqGap: + Task { await self.refresh() } + case let .snapshot(hello): + self.applyPresence(hello.snapshot.presence) + default: + break + } + } + + func refresh() async { + if self.isLoading { return } + self.statusMessage = nil + self.isLoading = true + defer { self.isLoading = false } + do { + PresenceReporter.shared.sendImmediate(reason: "instances-refresh") + let data = try await ControlChannel.shared.request(method: "system-presence") + self.lastPayload = data + if data.isEmpty { + self.logger.error("instances fetch returned empty payload") + self.instances = [self.localFallbackInstance(reason: "no presence payload")] + self.lastError = nil + self.statusMessage = "No presence payload from gateway; showing local fallback + health probe." + await self.probeHealthIfNeeded(reason: "no payload") + return + } + let decoded = try JSONDecoder().decode([PresenceEntry].self, from: data) + let withIDs = self.normalizePresence(decoded) + if withIDs.isEmpty { + self.instances = [self.localFallbackInstance(reason: "no presence entries")] + self.lastError = nil + self.statusMessage = "Presence list was empty; showing local fallback + health probe." + await self.probeHealthIfNeeded(reason: "empty list") + } else { + self.instances = withIDs + self.lastError = nil + self.statusMessage = nil + } + } catch { + self.logger.error( + """ + instances fetch failed: \(error.localizedDescription, privacy: .public) \ + len=\(self.lastPayload?.count ?? 0, privacy: .public) \ + utf8=\(self.snippet(self.lastPayload), privacy: .public) + """) + self.instances = [self.localFallbackInstance(reason: "presence decode failed")] + self.lastError = nil + self.statusMessage = "Presence data invalid; showing local fallback + health probe." + await self.probeHealthIfNeeded(reason: "decode failed") + } + } + + private func localFallbackInstance(reason: String) -> InstanceInfo { + let host = Host.current().localizedName ?? "this-mac" + let ip = Self.primaryIPv4Address() + let version = Bundle.main.object(forInfoDictionaryKey: "CFBundleShortVersionString") as? String + let osVersion = ProcessInfo.processInfo.operatingSystemVersion + let platform = "macos \(osVersion.majorVersion).\(osVersion.minorVersion).\(osVersion.patchVersion)" + let text = "Local node: \(host)\(ip.map { " (\($0))" } ?? "") · app \(version ?? "dev")" + let ts = Date().timeIntervalSince1970 * 1000 + return InstanceInfo( + id: "local-\(host)", + host: host, + ip: ip, + version: version, + platform: platform, + deviceFamily: "Mac", + modelIdentifier: InstanceIdentity.modelIdentifier, + lastInputSeconds: Self.lastInputSeconds(), + mode: "local", + reason: reason, + text: text, + ts: ts) + } + + private static func lastInputSeconds() -> Int? { + let anyEvent = CGEventType(rawValue: UInt32.max) ?? .null + let seconds = CGEventSource.secondsSinceLastEventType(.combinedSessionState, eventType: anyEvent) + if seconds.isNaN || seconds.isInfinite || seconds < 0 { return nil } + return Int(seconds.rounded()) + } + + private static func primaryIPv4Address() -> String? { + var addrList: UnsafeMutablePointer? + guard getifaddrs(&addrList) == 0, let first = addrList else { return nil } + defer { freeifaddrs(addrList) } + + var fallback: String? + var en0: String? + + for ptr in sequence(first: first, next: { $0.pointee.ifa_next }) { + let flags = Int32(ptr.pointee.ifa_flags) + let isUp = (flags & IFF_UP) != 0 + let isLoopback = (flags & IFF_LOOPBACK) != 0 + let name = String(cString: ptr.pointee.ifa_name) + let family = ptr.pointee.ifa_addr.pointee.sa_family + if !isUp || isLoopback || family != UInt8(AF_INET) { continue } + + var addr = ptr.pointee.ifa_addr.pointee + var buffer = [CChar](repeating: 0, count: Int(NI_MAXHOST)) + let result = getnameinfo( + &addr, + socklen_t(ptr.pointee.ifa_addr.pointee.sa_len), + &buffer, + socklen_t(buffer.count), + nil, + 0, + NI_NUMERICHOST) + guard result == 0 else { continue } + let len = buffer.prefix { $0 != 0 } + let bytes = len.map { UInt8(bitPattern: $0) } + guard let ip = String(bytes: bytes, encoding: .utf8) else { continue } + + if name == "en0" { en0 = ip; break } + if fallback == nil { fallback = ip } + } + + return en0 ?? fallback + } + + // MARK: - Helpers + + /// Keep the last raw payload for logging. + private var lastPayload: Data? + + private func snippet(_ data: Data?, limit: Int = 256) -> String { + guard let data else { return "" } + if data.isEmpty { return "" } + let prefix = data.prefix(limit) + if let asString = String(data: prefix, encoding: .utf8) { + return asString.replacingOccurrences(of: "\n", with: " ") + } + return "<\(data.count) bytes non-utf8>" + } + + private func probeHealthIfNeeded(reason: String? = nil) async { + do { + let data = try await ControlChannel.shared.health(timeout: 8) + guard let snap = decodeHealthSnapshot(from: data) else { return } + let linkId = snap.channelOrder?.first(where: { + if let summary = snap.channels[$0] { return summary.linked != nil } + return false + }) ?? snap.channels.keys.first(where: { + if let summary = snap.channels[$0] { return summary.linked != nil } + return false + }) + let linked = linkId.flatMap { snap.channels[$0]?.linked } ?? false + let linkLabel = + linkId.flatMap { snap.channelLabels?[$0] } ?? + linkId?.capitalized ?? + "channel" + let entry = InstanceInfo( + id: "health-\(snap.ts)", + host: "gateway (health)", + ip: nil, + version: nil, + platform: nil, + deviceFamily: nil, + modelIdentifier: nil, + lastInputSeconds: nil, + mode: "health", + reason: "health probe", + text: "Health ok · \(linkLabel) linked=\(linked)", + ts: snap.ts) + if !self.instances.contains(where: { $0.id == entry.id }) { + self.instances.insert(entry, at: 0) + } + self.lastError = nil + self.statusMessage = + "Presence unavailable (\(reason ?? "refresh")); showing health probe + local fallback." + } catch { + self.logger.error("instances health probe failed: \(error.localizedDescription, privacy: .public)") + if let reason { + self.statusMessage = + "Presence unavailable (\(reason)), health probe failed: \(error.localizedDescription)" + } + } + } + + private func decodeAndApplyPresenceData(_ data: Data) { + do { + let decoded = try JSONDecoder().decode([PresenceEntry].self, from: data) + self.applyPresence(decoded) + } catch { + self.logger.error("presence decode from event failed: \(error.localizedDescription, privacy: .public)") + self.lastError = error.localizedDescription + } + } + + func handlePresenceEventPayload(_ payload: OpenClawProtocol.AnyCodable) { + do { + let wrapper = try GatewayPayloadDecoding.decode(payload, as: PresenceEventPayload.self) + self.applyPresence(wrapper.presence) + } catch { + self.logger.error("presence event decode failed: \(error.localizedDescription, privacy: .public)") + self.lastError = error.localizedDescription + } + } + + private func normalizePresence(_ entries: [PresenceEntry]) -> [InstanceInfo] { + entries.map { entry -> InstanceInfo in + let key = entry.instanceid ?? entry.host ?? entry.ip ?? entry.text ?? "entry-\(entry.ts)" + return InstanceInfo( + id: key, + host: entry.host, + ip: entry.ip, + version: entry.version, + platform: entry.platform, + deviceFamily: entry.devicefamily, + modelIdentifier: entry.modelidentifier, + lastInputSeconds: entry.lastinputseconds, + mode: entry.mode, + reason: entry.reason, + text: entry.text ?? "Unnamed node", + ts: Double(entry.ts)) + } + } + + private func applyPresence(_ entries: [PresenceEntry]) { + let withIDs = self.normalizePresence(entries) + self.notifyOnNodeLogin(withIDs) + self.lastPresenceById = Dictionary(uniqueKeysWithValues: withIDs.map { ($0.id, $0) }) + self.instances = withIDs + self.statusMessage = nil + self.lastError = nil + } + + private func notifyOnNodeLogin(_ instances: [InstanceInfo]) { + for inst in instances { + guard let reason = inst.reason?.trimmingCharacters(in: .whitespacesAndNewlines) else { continue } + guard reason == "node-connected" else { continue } + if let mode = inst.mode?.lowercased(), mode == "local" { continue } + + let previous = self.lastPresenceById[inst.id] + if previous?.reason == "node-connected", previous?.ts == inst.ts { continue } + + let lastNotified = self.lastLoginNotifiedAtMs[inst.id] ?? 0 + if inst.ts <= lastNotified { continue } + self.lastLoginNotifiedAtMs[inst.id] = inst.ts + + let name = inst.host?.trimmingCharacters(in: .whitespacesAndNewlines) + let device = name?.isEmpty == false ? name! : inst.id + Task { @MainActor in + _ = await NotificationManager().send( + title: "Node connected", + body: device, + sound: nil, + priority: .active) + } + } + } +} + +extension InstancesStore { + static func preview(instances: [InstanceInfo] = [ + InstanceInfo( + id: "local", + host: "steipete-mac", + ip: "10.0.0.12", + version: "1.2.3", + platform: "macos 26.2.0", + deviceFamily: "Mac", + modelIdentifier: "Mac16,6", + lastInputSeconds: 12, + mode: "local", + reason: "preview", + text: "Local node: steipete-mac (10.0.0.12) · app 1.2.3", + ts: Date().timeIntervalSince1970 * 1000), + InstanceInfo( + id: "gateway", + host: "gateway", + ip: "100.64.0.2", + version: "1.2.3", + platform: "linux 6.6.0", + deviceFamily: "Linux", + modelIdentifier: "x86_64", + lastInputSeconds: 45, + mode: "remote", + reason: "preview", + text: "Gateway node · tunnel ok", + ts: Date().timeIntervalSince1970 * 1000 - 45000), + ]) -> InstancesStore { + let store = InstancesStore(isPreview: true) + store.instances = instances + store.statusMessage = "Preview data" + return store + } +} diff --git a/apps/macos/Sources/OpenClaw/LaunchAgentManager.swift b/apps/macos/Sources/OpenClaw/LaunchAgentManager.swift new file mode 100644 index 0000000000000000000000000000000000000000..af318b330d40b37ce06303bc11ab3cac1256ed07 --- /dev/null +++ b/apps/macos/Sources/OpenClaw/LaunchAgentManager.swift @@ -0,0 +1,78 @@ +import Foundation + +enum LaunchAgentManager { + private static var plistURL: URL { + FileManager().homeDirectoryForCurrentUser + .appendingPathComponent("Library/LaunchAgents/ai.openclaw.mac.plist") + } + + static func status() async -> Bool { + guard FileManager().fileExists(atPath: self.plistURL.path) else { return false } + let result = await self.runLaunchctl(["print", "gui/\(getuid())/\(launchdLabel)"]) + return result == 0 + } + + static func set(enabled: Bool, bundlePath: String) async { + if enabled { + self.writePlist(bundlePath: bundlePath) + _ = await self.runLaunchctl(["bootout", "gui/\(getuid())/\(launchdLabel)"]) + _ = await self.runLaunchctl(["bootstrap", "gui/\(getuid())", self.plistURL.path]) + _ = await self.runLaunchctl(["kickstart", "-k", "gui/\(getuid())/\(launchdLabel)"]) + } else { + // Disable autostart going forward but leave the current app running. + // bootout would terminate the launchd job immediately (and crash the app if launched via agent). + try? FileManager().removeItem(at: self.plistURL) + } + } + + private static func writePlist(bundlePath: String) { + let plist = """ + + + + + Label + ai.openclaw.mac + ProgramArguments + + \(bundlePath)/Contents/MacOS/OpenClaw + + WorkingDirectory + \(FileManager().homeDirectoryForCurrentUser.path) + RunAtLoad + + KeepAlive + + EnvironmentVariables + + PATH + \(CommandResolver.preferredPaths().joined(separator: ":")) + + StandardOutPath + \(LogLocator.launchdLogPath) + StandardErrorPath + \(LogLocator.launchdLogPath) + + + """ + try? plist.write(to: self.plistURL, atomically: true, encoding: .utf8) + } + + @discardableResult + private static func runLaunchctl(_ args: [String]) async -> Int32 { + await Task.detached(priority: .utility) { () -> Int32 in + let process = Process() + process.launchPath = "/bin/launchctl" + process.arguments = args + let pipe = Pipe() + process.standardOutput = pipe + process.standardError = pipe + do { + _ = try process.runAndReadToEnd(from: pipe) + return process.terminationStatus + } catch { + return -1 + } + }.value + } +} diff --git a/apps/macos/Sources/OpenClaw/Launchctl.swift b/apps/macos/Sources/OpenClaw/Launchctl.swift new file mode 100644 index 0000000000000000000000000000000000000000..cc50fd48ac7504eebd1dd7637b69f574847f6ebe --- /dev/null +++ b/apps/macos/Sources/OpenClaw/Launchctl.swift @@ -0,0 +1,87 @@ +import Foundation + +enum Launchctl { + struct Result: Sendable { + let status: Int32 + let output: String + } + + @discardableResult + static func run(_ args: [String]) async -> Result { + await Task.detached(priority: .utility) { () -> Result in + let process = Process() + process.launchPath = "/bin/launchctl" + process.arguments = args + let pipe = Pipe() + process.standardOutput = pipe + process.standardError = pipe + do { + let data = try process.runAndReadToEnd(from: pipe) + let output = String(data: data, encoding: .utf8) ?? "" + return Result(status: process.terminationStatus, output: output) + } catch { + return Result(status: -1, output: error.localizedDescription) + } + }.value + } +} + +struct LaunchAgentPlistSnapshot: Equatable, Sendable { + let programArguments: [String] + let environment: [String: String] + let stdoutPath: String? + let stderrPath: String? + + let port: Int? + let bind: String? + let token: String? + let password: String? +} + +enum LaunchAgentPlist { + static func snapshot(url: URL) -> LaunchAgentPlistSnapshot? { + guard let data = try? Data(contentsOf: url) else { return nil } + let rootAny: Any + do { + rootAny = try PropertyListSerialization.propertyList( + from: data, + options: [], + format: nil) + } catch { + return nil + } + guard let root = rootAny as? [String: Any] else { return nil } + let programArguments = root["ProgramArguments"] as? [String] ?? [] + let env = root["EnvironmentVariables"] as? [String: String] ?? [:] + let stdoutPath = (root["StandardOutPath"] as? String)? + .trimmingCharacters(in: .whitespacesAndNewlines).nonEmpty + let stderrPath = (root["StandardErrorPath"] as? String)? + .trimmingCharacters(in: .whitespacesAndNewlines).nonEmpty + let port = Self.extractFlagInt(programArguments, flag: "--port") + let bind = Self.extractFlagString(programArguments, flag: "--bind")?.lowercased() + let token = env["OPENCLAW_GATEWAY_TOKEN"]?.trimmingCharacters(in: .whitespacesAndNewlines).nonEmpty + let password = env["OPENCLAW_GATEWAY_PASSWORD"]?.trimmingCharacters(in: .whitespacesAndNewlines).nonEmpty + return LaunchAgentPlistSnapshot( + programArguments: programArguments, + environment: env, + stdoutPath: stdoutPath, + stderrPath: stderrPath, + port: port, + bind: bind, + token: token, + password: password) + } + + private static func extractFlagInt(_ args: [String], flag: String) -> Int? { + guard let raw = self.extractFlagString(args, flag: flag) else { return nil } + return Int(raw) + } + + private static func extractFlagString(_ args: [String], flag: String) -> String? { + guard let idx = args.firstIndex(of: flag) else { return nil } + let valueIdx = args.index(after: idx) + guard valueIdx < args.endIndex else { return nil } + let token = args[valueIdx].trimmingCharacters(in: .whitespacesAndNewlines) + return token.isEmpty ? nil : token + } +} diff --git a/apps/macos/Sources/OpenClaw/LaunchdManager.swift b/apps/macos/Sources/OpenClaw/LaunchdManager.swift new file mode 100644 index 0000000000000000000000000000000000000000..961246f194b50966a3eb6f464f65b1f9ac9c252d --- /dev/null +++ b/apps/macos/Sources/OpenClaw/LaunchdManager.swift @@ -0,0 +1,20 @@ +import Foundation + +enum LaunchdManager { + private static func runLaunchctl(_ args: [String]) { + let process = Process() + process.launchPath = "/bin/launchctl" + process.arguments = args + try? process.run() + } + + static func startOpenClaw() { + let userTarget = "gui/\(getuid())/\(launchdLabel)" + self.runLaunchctl(["kickstart", "-k", userTarget]) + } + + static func stopOpenClaw() { + let userTarget = "gui/\(getuid())/\(launchdLabel)" + self.runLaunchctl(["stop", userTarget]) + } +} diff --git a/apps/macos/Sources/OpenClaw/LogLocator.swift b/apps/macos/Sources/OpenClaw/LogLocator.swift new file mode 100644 index 0000000000000000000000000000000000000000..927b7892a28001bfe8bab9270ec04a8eeabaf36d --- /dev/null +++ b/apps/macos/Sources/OpenClaw/LogLocator.swift @@ -0,0 +1,60 @@ +import Foundation + +enum LogLocator { + private static var logDir: URL { + if let override = ProcessInfo.processInfo.environment["OPENCLAW_LOG_DIR"], + !override.isEmpty + { + return URL(fileURLWithPath: override) + } + let preferred = URL(fileURLWithPath: "/tmp/openclaw") + return preferred + } + + private static var stdoutLog: URL { + logDir.appendingPathComponent("openclaw-stdout.log") + } + + private static var gatewayLog: URL { + logDir.appendingPathComponent("openclaw-gateway.log") + } + + private static func ensureLogDirExists() { + try? FileManager().createDirectory(at: self.logDir, withIntermediateDirectories: true) + } + + private static func modificationDate(for url: URL) -> Date { + (try? url.resourceValues(forKeys: [.contentModificationDateKey]).contentModificationDate) ?? .distantPast + } + + /// Returns the newest log file under /tmp/openclaw/ (rolling or stdout), or nil if none exist. + static func bestLogFile() -> URL? { + self.ensureLogDirExists() + let fm = FileManager() + let files = (try? fm.contentsOfDirectory( + at: self.logDir, + includingPropertiesForKeys: [.contentModificationDateKey], + options: [.skipsHiddenFiles])) ?? [] + + let prefixes = ["openclaw"] + return files + .filter { file in + prefixes.contains { file.lastPathComponent.hasPrefix($0) } && file.pathExtension == "log" + } + .max { lhs, rhs in + self.modificationDate(for: lhs) < self.modificationDate(for: rhs) + } + } + + /// Path to use for launchd stdout/err. + static var launchdLogPath: String { + self.ensureLogDirExists() + return stdoutLog.path + } + + /// Path to use for the Gateway launchd job stdout/err. + static var launchdGatewayLogPath: String { + self.ensureLogDirExists() + return gatewayLog.path + } +} diff --git a/apps/macos/Sources/OpenClaw/Logging/OpenClawLogging.swift b/apps/macos/Sources/OpenClaw/Logging/OpenClawLogging.swift new file mode 100644 index 0000000000000000000000000000000000000000..bd46a8e6ff095369954041387d3b0690c773d929 --- /dev/null +++ b/apps/macos/Sources/OpenClaw/Logging/OpenClawLogging.swift @@ -0,0 +1,230 @@ +import Foundation +@_exported import Logging +import os +import OSLog + +typealias Logger = Logging.Logger + +enum AppLogSettings { + static let logLevelKey = appLogLevelKey + + static func logLevel() -> Logger.Level { + if let raw = UserDefaults.standard.string(forKey: self.logLevelKey), + let level = Logger.Level(rawValue: raw) + { + return level + } + return .info + } + + static func setLogLevel(_ level: Logger.Level) { + UserDefaults.standard.set(level.rawValue, forKey: self.logLevelKey) + } + + static func fileLoggingEnabled() -> Bool { + UserDefaults.standard.bool(forKey: debugFileLogEnabledKey) + } +} + +enum AppLogLevel: String, CaseIterable, Identifiable { + case trace + case debug + case info + case notice + case warning + case error + case critical + + static let `default`: AppLogLevel = .info + + var id: String { self.rawValue } + + var title: String { + switch self { + case .trace: "Trace" + case .debug: "Debug" + case .info: "Info" + case .notice: "Notice" + case .warning: "Warning" + case .error: "Error" + case .critical: "Critical" + } + } +} + +enum OpenClawLogging { + private static let labelSeparator = "::" + + private static let didBootstrap: Void = { + LoggingSystem.bootstrap { label in + let (subsystem, category) = Self.parseLabel(label) + let osHandler = OpenClawOSLogHandler(subsystem: subsystem, category: category) + let fileHandler = OpenClawFileLogHandler(label: label) + return MultiplexLogHandler([osHandler, fileHandler]) + } + }() + + static func bootstrapIfNeeded() { + _ = self.didBootstrap + } + + static func makeLabel(subsystem: String, category: String) -> String { + "\(subsystem)\(self.labelSeparator)\(category)" + } + + static func parseLabel(_ label: String) -> (String, String) { + guard let range = label.range(of: labelSeparator) else { + return ("ai.openclaw", label) + } + let subsystem = String(label[.. Logger.Metadata.Value? { + get { self.metadata[key] } + set { self.metadata[key] = newValue } + } + + func log( + level: Logger.Level, + message: Logger.Message, + metadata: Logger.Metadata?, + source: String, + file: String, + function: String, + line: UInt) + { + let merged = Self.mergeMetadata(self.metadata, metadata) + let rendered = Self.renderMessage(message, metadata: merged) + self.osLogger.log(level: Self.osLogType(for: level), "\(rendered, privacy: .public)") + } + + private static func osLogType(for level: Logger.Level) -> OSLogType { + switch level { + case .trace, .debug: + .debug + case .info, .notice: + .info + case .warning: + .default + case .error: + .error + case .critical: + .fault + } + } + + private static func mergeMetadata( + _ base: Logger.Metadata, + _ extra: Logger.Metadata?) -> Logger.Metadata + { + guard let extra else { return base } + return base.merging(extra, uniquingKeysWith: { _, new in new }) + } + + private static func renderMessage(_ message: Logger.Message, metadata: Logger.Metadata) -> String { + guard !metadata.isEmpty else { return message.description } + let meta = metadata + .sorted(by: { $0.key < $1.key }) + .map { "\($0.key)=\(self.stringify($0.value))" } + .joined(separator: " ") + return "\(message.description) [\(meta)]" + } + + private static func stringify(_ value: Logger.Metadata.Value) -> String { + switch value { + case let .string(text): + text + case let .stringConvertible(value): + String(describing: value) + case let .array(values): + "[" + values.map { self.stringify($0) }.joined(separator: ",") + "]" + case let .dictionary(entries): + "{" + entries.map { "\($0.key)=\(self.stringify($0.value))" }.joined(separator: ",") + "}" + } + } +} + +struct OpenClawFileLogHandler: LogHandler { + let label: String + var metadata: Logger.Metadata = [:] + + var logLevel: Logger.Level { + get { AppLogSettings.logLevel() } + set { AppLogSettings.setLogLevel(newValue) } + } + + subscript(metadataKey key: String) -> Logger.Metadata.Value? { + get { self.metadata[key] } + set { self.metadata[key] = newValue } + } + + func log( + level: Logger.Level, + message: Logger.Message, + metadata: Logger.Metadata?, + source: String, + file: String, + function: String, + line: UInt) + { + guard AppLogSettings.fileLoggingEnabled() else { return } + let (subsystem, category) = OpenClawLogging.parseLabel(self.label) + var fields: [String: String] = [ + "subsystem": subsystem, + "category": category, + "level": level.rawValue, + "source": source, + "file": file, + "function": function, + "line": "\(line)", + ] + let merged = self.metadata.merging(metadata ?? [:], uniquingKeysWith: { _, new in new }) + for (key, value) in merged { + fields["meta.\(key)"] = Self.stringify(value) + } + DiagnosticsFileLog.shared.log(category: category, event: message.description, fields: fields) + } + + private static func stringify(_ value: Logger.Metadata.Value) -> String { + switch value { + case let .string(text): + text + case let .stringConvertible(value): + String(describing: value) + case let .array(values): + "[" + values.map { self.stringify($0) }.joined(separator: ",") + "]" + case let .dictionary(entries): + "{" + entries.map { "\($0.key)=\(self.stringify($0.value))" }.joined(separator: ",") + "}" + } + } +} diff --git a/apps/macos/Sources/OpenClaw/MenuBar.swift b/apps/macos/Sources/OpenClaw/MenuBar.swift new file mode 100644 index 0000000000000000000000000000000000000000..e2af092323f37f0e86bca90754b012aa649b26ae --- /dev/null +++ b/apps/macos/Sources/OpenClaw/MenuBar.swift @@ -0,0 +1,471 @@ +import AppKit +import Darwin +import Foundation +import MenuBarExtraAccess +import Observation +import OSLog +import Security +import SwiftUI + +@main +struct OpenClawApp: App { + @NSApplicationDelegateAdaptor(AppDelegate.self) private var delegate + @State private var state: AppState + private static let logger = Logger(subsystem: "ai.openclaw", category: "app") + private let gatewayManager = GatewayProcessManager.shared + private let controlChannel = ControlChannel.shared + private let activityStore = WorkActivityStore.shared + private let connectivityCoordinator = GatewayConnectivityCoordinator.shared + @State private var statusItem: NSStatusItem? + @State private var isMenuPresented = false + @State private var isPanelVisible = false + @State private var tailscaleService = TailscaleService.shared + + @MainActor + private func updateStatusHighlight() { + self.statusItem?.button?.highlight(self.isPanelVisible) + } + + @MainActor + private func updateHoverHUDSuppression() { + HoverHUDController.shared.setSuppressed(self.isMenuPresented || self.isPanelVisible) + } + + init() { + OpenClawLogging.bootstrapIfNeeded() + Self.applyAttachOnlyOverrideIfNeeded() + _state = State(initialValue: AppStateStore.shared) + } + + var body: some Scene { + MenuBarExtra { MenuContent(state: self.state, updater: self.delegate.updaterController) } label: { + CritterStatusLabel( + isPaused: self.state.isPaused, + isSleeping: self.isGatewaySleeping, + isWorking: self.state.isWorking, + earBoostActive: self.state.earBoostActive, + blinkTick: self.state.blinkTick, + sendCelebrationTick: self.state.sendCelebrationTick, + gatewayStatus: self.gatewayManager.status, + animationsEnabled: self.state.iconAnimationsEnabled && !self.isGatewaySleeping, + iconState: self.effectiveIconState) + } + .menuBarExtraStyle(.menu) + .menuBarExtraAccess(isPresented: self.$isMenuPresented) { item in + self.statusItem = item + MenuSessionsInjector.shared.install(into: item) + self.applyStatusItemAppearance(paused: self.state.isPaused, sleeping: self.isGatewaySleeping) + self.installStatusItemMouseHandler(for: item) + self.updateHoverHUDSuppression() + } + .onChange(of: self.state.isPaused) { _, paused in + self.applyStatusItemAppearance(paused: paused, sleeping: self.isGatewaySleeping) + if self.state.connectionMode == .local { + self.gatewayManager.setActive(!paused) + } else { + self.gatewayManager.stop() + } + } + .onChange(of: self.controlChannel.state) { _, _ in + self.applyStatusItemAppearance(paused: self.state.isPaused, sleeping: self.isGatewaySleeping) + } + .onChange(of: self.gatewayManager.status) { _, _ in + self.applyStatusItemAppearance(paused: self.state.isPaused, sleeping: self.isGatewaySleeping) + } + .onChange(of: self.state.connectionMode) { _, mode in + Task { await ConnectionModeCoordinator.shared.apply(mode: mode, paused: self.state.isPaused) } + CLIInstallPrompter.shared.checkAndPromptIfNeeded(reason: "connection-mode") + } + + Settings { + SettingsRootView(state: self.state, updater: self.delegate.updaterController) + .frame(width: SettingsTab.windowWidth, height: SettingsTab.windowHeight, alignment: .topLeading) + .environment(self.tailscaleService) + } + .defaultSize(width: SettingsTab.windowWidth, height: SettingsTab.windowHeight) + .windowResizability(.contentSize) + .onChange(of: self.isMenuPresented) { _, _ in + self.updateStatusHighlight() + self.updateHoverHUDSuppression() + } + } + + private func applyStatusItemAppearance(paused: Bool, sleeping: Bool) { + self.statusItem?.button?.appearsDisabled = paused || sleeping + } + + private static func applyAttachOnlyOverrideIfNeeded() { + let args = CommandLine.arguments + guard args.contains("--attach-only") || args.contains("--no-launchd") else { return } + if let error = GatewayLaunchAgentManager.setLaunchAgentWriteDisabled(true) { + Self.logger.error("attach-only flag failed: \(error, privacy: .public)") + return + } + Task { + _ = await GatewayLaunchAgentManager.set( + enabled: false, + bundlePath: Bundle.main.bundlePath, + port: GatewayEnvironment.gatewayPort()) + } + Self.logger.info("attach-only flag enabled") + } + + private var isGatewaySleeping: Bool { + if self.state.isPaused { return false } + switch self.state.connectionMode { + case .unconfigured: + return true + case .remote: + if case .connected = self.controlChannel.state { return false } + return true + case .local: + switch self.gatewayManager.status { + case .running, .starting, .attachedExisting: + if case .connected = self.controlChannel.state { return false } + return true + case .failed, .stopped: + return true + } + } + } + + @MainActor + private func installStatusItemMouseHandler(for item: NSStatusItem) { + guard let button = item.button else { return } + if button.subviews.contains(where: { $0 is StatusItemMouseHandlerView }) { return } + + WebChatManager.shared.onPanelVisibilityChanged = { [self] visible in + self.isPanelVisible = visible + self.updateStatusHighlight() + self.updateHoverHUDSuppression() + } + CanvasManager.shared.onPanelVisibilityChanged = { [self] visible in + self.state.canvasPanelVisible = visible + } + CanvasManager.shared.defaultAnchorProvider = { [self] in self.statusButtonScreenFrame() } + + let handler = StatusItemMouseHandlerView() + handler.translatesAutoresizingMaskIntoConstraints = false + handler.onLeftClick = { [self] in + HoverHUDController.shared.dismiss(reason: "statusItemClick") + self.toggleWebChatPanel() + } + handler.onRightClick = { [self] in + HoverHUDController.shared.dismiss(reason: "statusItemRightClick") + WebChatManager.shared.closePanel() + self.isMenuPresented = true + self.updateStatusHighlight() + } + handler.onHoverChanged = { [self] inside in + HoverHUDController.shared.statusItemHoverChanged( + inside: inside, + anchorProvider: { [self] in self.statusButtonScreenFrame() }) + } + + button.addSubview(handler) + NSLayoutConstraint.activate([ + handler.leadingAnchor.constraint(equalTo: button.leadingAnchor), + handler.trailingAnchor.constraint(equalTo: button.trailingAnchor), + handler.topAnchor.constraint(equalTo: button.topAnchor), + handler.bottomAnchor.constraint(equalTo: button.bottomAnchor), + ]) + } + + @MainActor + private func toggleWebChatPanel() { + HoverHUDController.shared.setSuppressed(true) + self.isMenuPresented = false + Task { @MainActor in + let sessionKey = await WebChatManager.shared.preferredSessionKey() + WebChatManager.shared.togglePanel( + sessionKey: sessionKey, + anchorProvider: { [self] in self.statusButtonScreenFrame() }) + } + } + + @MainActor + private func statusButtonScreenFrame() -> NSRect? { + guard let button = self.statusItem?.button, let window = button.window else { return nil } + let inWindow = button.convert(button.bounds, to: nil) + return window.convertToScreen(inWindow) + } + + private var effectiveIconState: IconState { + let selection = self.state.iconOverride + if selection == .system { + return self.activityStore.iconState + } + let overrideState = selection.toIconState() + switch overrideState { + case let .workingMain(kind): return .overridden(kind) + case let .workingOther(kind): return .overridden(kind) + case .idle: return .idle + case let .overridden(kind): return .overridden(kind) + } + } +} + +/// Transparent overlay that intercepts clicks without stealing MenuBarExtra ownership. +private final class StatusItemMouseHandlerView: NSView { + var onLeftClick: (() -> Void)? + var onRightClick: (() -> Void)? + var onHoverChanged: ((Bool) -> Void)? + private var tracking: NSTrackingArea? + + override func mouseDown(with event: NSEvent) { + if let onLeftClick { + onLeftClick() + } else { + super.mouseDown(with: event) + } + } + + override func rightMouseDown(with event: NSEvent) { + self.onRightClick?() + // Do not call super; menu will be driven by isMenuPresented binding. + } + + override func updateTrackingAreas() { + super.updateTrackingAreas() + if let tracking { + self.removeTrackingArea(tracking) + } + let options: NSTrackingArea.Options = [ + .mouseEnteredAndExited, + .activeAlways, + .inVisibleRect, + ] + let area = NSTrackingArea(rect: self.bounds, options: options, owner: self, userInfo: nil) + self.addTrackingArea(area) + self.tracking = area + } + + override func mouseEntered(with event: NSEvent) { + self.onHoverChanged?(true) + } + + override func mouseExited(with event: NSEvent) { + self.onHoverChanged?(false) + } +} + +@MainActor +final class AppDelegate: NSObject, NSApplicationDelegate { + private var state: AppState? + private let webChatAutoLogger = Logger(subsystem: "ai.openclaw", category: "Chat") + let updaterController: UpdaterProviding = makeUpdaterController() + + func application(_: NSApplication, open urls: [URL]) { + Task { @MainActor in + for url in urls { + await DeepLinkHandler.shared.handle(url: url) + } + } + } + + @MainActor + func applicationDidFinishLaunching(_ notification: Notification) { + if self.isDuplicateInstance() { + NSApp.terminate(nil) + return + } + self.state = AppStateStore.shared + AppActivationPolicy.apply(showDockIcon: self.state?.showDockIcon ?? false) + if let state { + Task { await ConnectionModeCoordinator.shared.apply(mode: state.connectionMode, paused: state.isPaused) } + } + TerminationSignalWatcher.shared.start() + NodePairingApprovalPrompter.shared.start() + DevicePairingApprovalPrompter.shared.start() + ExecApprovalsPromptServer.shared.start() + ExecApprovalsGatewayPrompter.shared.start() + MacNodeModeCoordinator.shared.start() + VoiceWakeGlobalSettingsSync.shared.start() + Task { PresenceReporter.shared.start() } + Task { await HealthStore.shared.refresh(onDemand: true) } + Task { await PortGuardian.shared.sweep(mode: AppStateStore.shared.connectionMode) } + Task { await PeekabooBridgeHostCoordinator.shared.setEnabled(AppStateStore.shared.peekabooBridgeEnabled) } + self.scheduleFirstRunOnboardingIfNeeded() + DispatchQueue.main.asyncAfter(deadline: .now() + 1.0) { + CLIInstallPrompter.shared.checkAndPromptIfNeeded(reason: "launch") + } + + // Developer/testing helper: auto-open chat when launched with --chat (or legacy --webchat). + if CommandLine.arguments.contains("--chat") || CommandLine.arguments.contains("--webchat") { + self.webChatAutoLogger.debug("Auto-opening chat via CLI flag") + Task { @MainActor in + let sessionKey = await WebChatManager.shared.preferredSessionKey() + WebChatManager.shared.show(sessionKey: sessionKey) + } + } + } + + func applicationWillTerminate(_ notification: Notification) { + PresenceReporter.shared.stop() + NodePairingApprovalPrompter.shared.stop() + DevicePairingApprovalPrompter.shared.stop() + ExecApprovalsPromptServer.shared.stop() + ExecApprovalsGatewayPrompter.shared.stop() + MacNodeModeCoordinator.shared.stop() + TerminationSignalWatcher.shared.stop() + VoiceWakeGlobalSettingsSync.shared.stop() + WebChatManager.shared.close() + WebChatManager.shared.resetTunnels() + Task { await RemoteTunnelManager.shared.stopAll() } + Task { await GatewayConnection.shared.shutdown() } + Task { await PeekabooBridgeHostCoordinator.shared.stop() } + } + + @MainActor + private func scheduleFirstRunOnboardingIfNeeded() { + let seenVersion = UserDefaults.standard.integer(forKey: onboardingVersionKey) + let shouldShow = seenVersion < currentOnboardingVersion || !AppStateStore.shared.onboardingSeen + guard shouldShow else { return } + DispatchQueue.main.asyncAfter(deadline: .now() + 0.6) { + OnboardingController.shared.show() + } + } + + private func isDuplicateInstance() -> Bool { + guard let bundleID = Bundle.main.bundleIdentifier else { return false } + let running = NSWorkspace.shared.runningApplications.filter { $0.bundleIdentifier == bundleID } + return running.count > 1 + } +} + +// MARK: - Sparkle updater (disabled for unsigned/dev builds) + +@MainActor +protocol UpdaterProviding: AnyObject { + var automaticallyChecksForUpdates: Bool { get set } + var automaticallyDownloadsUpdates: Bool { get set } + var isAvailable: Bool { get } + var updateStatus: UpdateStatus { get } + func checkForUpdates(_ sender: Any?) +} + +// No-op updater used for debug/dev runs to suppress Sparkle dialogs. +final class DisabledUpdaterController: UpdaterProviding { + var automaticallyChecksForUpdates: Bool = false + var automaticallyDownloadsUpdates: Bool = false + let isAvailable: Bool = false + let updateStatus = UpdateStatus() + func checkForUpdates(_: Any?) {} +} + +@MainActor +@Observable +final class UpdateStatus { + static let disabled = UpdateStatus() + var isUpdateReady: Bool + + init(isUpdateReady: Bool = false) { + self.isUpdateReady = isUpdateReady + } +} + +#if canImport(Sparkle) +import Sparkle + +@MainActor +final class SparkleUpdaterController: NSObject, UpdaterProviding { + private lazy var controller = SPUStandardUpdaterController( + startingUpdater: false, + updaterDelegate: self, + userDriverDelegate: nil) + let updateStatus = UpdateStatus() + + init(savedAutoUpdate: Bool) { + super.init() + let updater = self.controller.updater + updater.automaticallyChecksForUpdates = savedAutoUpdate + updater.automaticallyDownloadsUpdates = savedAutoUpdate + self.controller.startUpdater() + } + + var automaticallyChecksForUpdates: Bool { + get { self.controller.updater.automaticallyChecksForUpdates } + set { self.controller.updater.automaticallyChecksForUpdates = newValue } + } + + var automaticallyDownloadsUpdates: Bool { + get { self.controller.updater.automaticallyDownloadsUpdates } + set { self.controller.updater.automaticallyDownloadsUpdates = newValue } + } + + var isAvailable: Bool { true } + + func checkForUpdates(_ sender: Any?) { + self.controller.checkForUpdates(sender) + } + + func updater(_ updater: SPUUpdater, didDownloadUpdate item: SUAppcastItem) { + self.updateStatus.isUpdateReady = true + } + + func updater(_ updater: SPUUpdater, failedToDownloadUpdate item: SUAppcastItem, error: Error) { + self.updateStatus.isUpdateReady = false + } + + func userDidCancelDownload(_ updater: SPUUpdater) { + self.updateStatus.isUpdateReady = false + } + + func updater( + _ updater: SPUUpdater, + userDidMakeChoice choice: SPUUserUpdateChoice, + forUpdate updateItem: SUAppcastItem, + state: SPUUserUpdateState) + { + switch choice { + case .install, .skip: + self.updateStatus.isUpdateReady = false + case .dismiss: + self.updateStatus.isUpdateReady = (state.stage == .downloaded) + @unknown default: + self.updateStatus.isUpdateReady = false + } + } +} + +extension SparkleUpdaterController: @preconcurrency SPUUpdaterDelegate {} + +private func isDeveloperIDSigned(bundleURL: URL) -> Bool { + var staticCode: SecStaticCode? + guard SecStaticCodeCreateWithPath(bundleURL as CFURL, SecCSFlags(), &staticCode) == errSecSuccess, + let code = staticCode + else { return false } + + var infoCF: CFDictionary? + guard SecCodeCopySigningInformation(code, SecCSFlags(rawValue: kSecCSSigningInformation), &infoCF) == errSecSuccess, + let info = infoCF as? [String: Any], + let certs = info[kSecCodeInfoCertificates as String] as? [SecCertificate], + let leaf = certs.first + else { + return false + } + + if let summary = SecCertificateCopySubjectSummary(leaf) as String? { + return summary.hasPrefix("Developer ID Application:") + } + return false +} + +@MainActor +private func makeUpdaterController() -> UpdaterProviding { + let bundleURL = Bundle.main.bundleURL + let isBundledApp = bundleURL.pathExtension == "app" + guard isBundledApp, isDeveloperIDSigned(bundleURL: bundleURL) else { return DisabledUpdaterController() } + + let defaults = UserDefaults.standard + let autoUpdateKey = "autoUpdateEnabled" + // Default to true; honor the user's last choice otherwise. + let savedAutoUpdate = (defaults.object(forKey: autoUpdateKey) as? Bool) ?? true + return SparkleUpdaterController(savedAutoUpdate: savedAutoUpdate) +} +#else +@MainActor +private func makeUpdaterController() -> UpdaterProviding { + DisabledUpdaterController() +} +#endif diff --git a/apps/macos/Sources/OpenClaw/MenuContentView.swift b/apps/macos/Sources/OpenClaw/MenuContentView.swift new file mode 100644 index 0000000000000000000000000000000000000000..6dec4d9362095e3adfd87f54b67a3cfd746c1d91 --- /dev/null +++ b/apps/macos/Sources/OpenClaw/MenuContentView.swift @@ -0,0 +1,595 @@ +import AppKit +import AVFoundation +import Foundation +import Observation +import SwiftUI + +/// Menu contents for the OpenClaw menu bar extra. +struct MenuContent: View { + @Bindable var state: AppState + let updater: UpdaterProviding? + @Bindable private var updateStatus: UpdateStatus + private let gatewayManager = GatewayProcessManager.shared + private let healthStore = HealthStore.shared + private let heartbeatStore = HeartbeatStore.shared + private let controlChannel = ControlChannel.shared + private let activityStore = WorkActivityStore.shared + @Bindable private var pairingPrompter = NodePairingApprovalPrompter.shared + @Bindable private var devicePairingPrompter = DevicePairingApprovalPrompter.shared + @Environment(\.openSettings) private var openSettings + @State private var availableMics: [AudioInputDevice] = [] + @State private var loadingMics = false + @State private var micObserver = AudioInputDeviceObserver() + @State private var micRefreshTask: Task? + @State private var browserControlEnabled = true + @AppStorage(cameraEnabledKey) private var cameraEnabled: Bool = false + @AppStorage(appLogLevelKey) private var appLogLevelRaw: String = AppLogLevel.default.rawValue + @AppStorage(debugFileLogEnabledKey) private var appFileLoggingEnabled: Bool = false + + init(state: AppState, updater: UpdaterProviding?) { + self._state = Bindable(wrappedValue: state) + self.updater = updater + self._updateStatus = Bindable(wrappedValue: updater?.updateStatus ?? UpdateStatus.disabled) + } + + private var execApprovalModeBinding: Binding { + Binding( + get: { self.state.execApprovalMode }, + set: { self.state.execApprovalMode = $0 }) + } + + var body: some View { + VStack(alignment: .leading, spacing: 8) { + Toggle(isOn: self.activeBinding) { + VStack(alignment: .leading, spacing: 2) { + Text(self.connectionLabel) + self.statusLine(label: self.healthStatus.label, color: self.healthStatus.color) + if self.pairingPrompter.pendingCount > 0 { + let repairCount = self.pairingPrompter.pendingRepairCount + let repairSuffix = repairCount > 0 ? " · \(repairCount) repair" : "" + self.statusLine( + label: "Pairing approval pending (\(self.pairingPrompter.pendingCount))\(repairSuffix)", + color: .orange) + } + if self.devicePairingPrompter.pendingCount > 0 { + let repairCount = self.devicePairingPrompter.pendingRepairCount + let repairSuffix = repairCount > 0 ? " · \(repairCount) repair" : "" + self.statusLine( + label: "Device pairing pending (\(self.devicePairingPrompter.pendingCount))\(repairSuffix)", + color: .orange) + } + } + } + .disabled(self.state.connectionMode == .unconfigured) + + Divider() + Toggle(isOn: self.heartbeatsBinding) { + HStack(spacing: 8) { + Label("Send Heartbeats", systemImage: "waveform.path.ecg") + Spacer(minLength: 0) + self.statusLine(label: self.heartbeatStatus.label, color: self.heartbeatStatus.color) + } + } + Toggle( + isOn: Binding( + get: { self.browserControlEnabled }, + set: { enabled in + self.browserControlEnabled = enabled + Task { await self.saveBrowserControlEnabled(enabled) } + })) { + Label("Browser Control", systemImage: "globe") + } + Toggle(isOn: self.$cameraEnabled) { + Label("Allow Camera", systemImage: "camera") + } + Picker(selection: self.execApprovalModeBinding) { + ForEach(ExecApprovalQuickMode.allCases) { mode in + Text(mode.title).tag(mode) + } + } label: { + Label("Exec Approvals", systemImage: "terminal") + } + Toggle(isOn: Binding(get: { self.state.canvasEnabled }, set: { self.state.canvasEnabled = $0 })) { + Label("Allow Canvas", systemImage: "rectangle.and.pencil.and.ellipsis") + } + .onChange(of: self.state.canvasEnabled) { _, enabled in + if !enabled { + CanvasManager.shared.hideAll() + } + } + Toggle(isOn: self.voiceWakeBinding) { + Label("Voice Wake", systemImage: "mic.fill") + } + .disabled(!voiceWakeSupported) + .opacity(voiceWakeSupported ? 1 : 0.5) + if self.showVoiceWakeMicPicker { + self.voiceWakeMicMenu + } + Divider() + Button { + Task { @MainActor in + await self.openDashboard() + } + } label: { + Label("Open Dashboard", systemImage: "gauge") + } + Button { + Task { @MainActor in + let sessionKey = await WebChatManager.shared.preferredSessionKey() + WebChatManager.shared.show(sessionKey: sessionKey) + } + } label: { + Label("Open Chat", systemImage: "bubble.left.and.bubble.right") + } + if self.state.canvasEnabled { + Button { + Task { @MainActor in + if self.state.canvasPanelVisible { + CanvasManager.shared.hideAll() + } else { + let sessionKey = await GatewayConnection.shared.mainSessionKey() + // Don't force a navigation on re-open: preserve the current web view state. + _ = try? CanvasManager.shared.show(sessionKey: sessionKey, path: nil) + } + } + } label: { + Label( + self.state.canvasPanelVisible ? "Close Canvas" : "Open Canvas", + systemImage: "rectangle.inset.filled.on.rectangle") + } + } + Button { + Task { await self.state.setTalkEnabled(!self.state.talkEnabled) } + } label: { + Label(self.state.talkEnabled ? "Stop Talk Mode" : "Talk Mode", systemImage: "waveform.circle.fill") + } + .disabled(!voiceWakeSupported) + .opacity(voiceWakeSupported ? 1 : 0.5) + Divider() + Button("Settings…") { self.open(tab: .general) } + .keyboardShortcut(",", modifiers: [.command]) + self.debugMenu + Button("About OpenClaw") { self.open(tab: .about) } + if let updater, updater.isAvailable, self.updateStatus.isUpdateReady { + Button("Update ready, restart now?") { updater.checkForUpdates(nil) } + } + Button("Quit") { NSApplication.shared.terminate(nil) } + } + .task(id: self.state.swabbleEnabled) { + if self.state.swabbleEnabled { + await self.loadMicrophones(force: true) + } + } + .task { + VoicePushToTalkHotkey.shared.setEnabled(voiceWakeSupported && self.state.voicePushToTalkEnabled) + } + .onChange(of: self.state.voicePushToTalkEnabled) { _, enabled in + VoicePushToTalkHotkey.shared.setEnabled(voiceWakeSupported && enabled) + } + .task(id: self.state.connectionMode) { + await self.loadBrowserControlEnabled() + } + .onAppear { + self.startMicObserver() + } + .onDisappear { + self.micRefreshTask?.cancel() + self.micRefreshTask = nil + self.micObserver.stop() + } + .task { @MainActor in + SettingsWindowOpener.shared.register(openSettings: self.openSettings) + } + } + + private var connectionLabel: String { + switch self.state.connectionMode { + case .unconfigured: + "OpenClaw Not Configured" + case .remote: + "Remote OpenClaw Active" + case .local: + "OpenClaw Active" + } + } + + private func loadBrowserControlEnabled() async { + let root = await ConfigStore.load() + let browser = root["browser"] as? [String: Any] + let enabled = browser?["enabled"] as? Bool ?? true + await MainActor.run { self.browserControlEnabled = enabled } + } + + private func saveBrowserControlEnabled(_ enabled: Bool) async { + let (success, _) = await MenuContent.buildAndSaveBrowserEnabled(enabled) + + if !success { + await self.loadBrowserControlEnabled() + } + } + + @MainActor + private static func buildAndSaveBrowserEnabled(_ enabled: Bool) async -> (Bool, ()) { + var root = await ConfigStore.load() + var browser = root["browser"] as? [String: Any] ?? [:] + browser["enabled"] = enabled + root["browser"] = browser + do { + try await ConfigStore.save(root) + return (true, ()) + } catch { + return (false, ()) + } + } + + @ViewBuilder + private var debugMenu: some View { + if self.state.debugPaneEnabled { + Menu("Debug") { + Button { + DebugActions.openConfigFolder() + } label: { + Label("Open Config Folder", systemImage: "folder") + } + Button { + Task { await DebugActions.runHealthCheckNow() } + } label: { + Label("Run Health Check Now", systemImage: "stethoscope") + } + Button { + Task { _ = await DebugActions.sendTestHeartbeat() } + } label: { + Label("Send Test Heartbeat", systemImage: "waveform.path.ecg") + } + if self.state.connectionMode == .remote { + Button { + Task { @MainActor in + let result = await DebugActions.resetGatewayTunnel() + self.presentDebugResult(result, title: "Remote Tunnel") + } + } label: { + Label("Reset Remote Tunnel", systemImage: "arrow.triangle.2.circlepath") + } + } + Button { + Task { _ = await DebugActions.toggleVerboseLoggingMain() } + } label: { + Label( + DebugActions.verboseLoggingEnabledMain + ? "Verbose Logging (Main): On" + : "Verbose Logging (Main): Off", + systemImage: "text.alignleft") + } + Menu { + Picker("Verbosity", selection: self.$appLogLevelRaw) { + ForEach(AppLogLevel.allCases) { level in + Text(level.title).tag(level.rawValue) + } + } + Toggle(isOn: self.$appFileLoggingEnabled) { + Label( + self.appFileLoggingEnabled + ? "File Logging: On" + : "File Logging: Off", + systemImage: "doc.text.magnifyingglass") + } + } label: { + Label("App Logging", systemImage: "doc.text") + } + Button { + DebugActions.openSessionStore() + } label: { + Label("Open Session Store", systemImage: "externaldrive") + } + Divider() + Button { + DebugActions.openAgentEventsWindow() + } label: { + Label("Open Agent Events…", systemImage: "bolt.horizontal.circle") + } + Button { + DebugActions.openLog() + } label: { + Label("Open Log", systemImage: "doc.text.magnifyingglass") + } + Button { + Task { _ = await DebugActions.sendDebugVoice() } + } label: { + Label("Send Debug Voice Text", systemImage: "waveform.circle") + } + Button { + Task { await DebugActions.sendTestNotification() } + } label: { + Label("Send Test Notification", systemImage: "bell") + } + Divider() + if self.state.connectionMode == .local { + Button { + DebugActions.restartGateway() + } label: { + Label("Restart Gateway", systemImage: "arrow.clockwise") + } + } + Button { + DebugActions.restartOnboarding() + } label: { + Label("Restart Onboarding", systemImage: "arrow.counterclockwise") + } + Button { + DebugActions.restartApp() + } label: { + Label("Restart App", systemImage: "arrow.triangle.2.circlepath") + } + } + } + } + + private func open(tab: SettingsTab) { + SettingsTabRouter.request(tab) + NSApp.activate(ignoringOtherApps: true) + self.openSettings() + DispatchQueue.main.async { + NotificationCenter.default.post(name: .openclawSelectSettingsTab, object: tab) + } + } + + @MainActor + private func openDashboard() async { + do { + let config = try await GatewayEndpointStore.shared.requireConfig() + let url = try GatewayEndpointStore.dashboardURL(for: config) + NSWorkspace.shared.open(url) + } catch { + let alert = NSAlert() + alert.messageText = "Dashboard unavailable" + alert.informativeText = error.localizedDescription + alert.runModal() + } + } + + private var healthStatus: (label: String, color: Color) { + if let activity = self.activityStore.current { + let color: Color = activity.role == .main ? .accentColor : .gray + let roleLabel = activity.role == .main ? "Main" : "Other" + let text = "\(roleLabel) · \(activity.label)" + return (text, color) + } + + let health = self.healthStore.state + let isRefreshing = self.healthStore.isRefreshing + let lastAge = self.healthStore.lastSuccess.map { age(from: $0) } + + if isRefreshing { + return ("Health check running…", health.tint) + } + + switch health { + case .ok: + let ageText = lastAge.map { " · checked \($0)" } ?? "" + return ("Health ok\(ageText)", .green) + case .linkingNeeded: + return ("Health: login required", .red) + case let .degraded(reason): + let detail = HealthStore.shared.degradedSummary ?? reason + let ageText = lastAge.map { " · checked \($0)" } ?? "" + return ("\(detail)\(ageText)", .orange) + case .unknown: + return ("Health pending", .secondary) + } + } + + private var heartbeatStatus: (label: String, color: Color) { + if case .degraded = self.controlChannel.state { + return ("Control channel disconnected", .red) + } else if let evt = self.heartbeatStore.lastEvent { + let ageText = age(from: Date(timeIntervalSince1970: evt.ts / 1000)) + switch evt.status { + case "sent": + return ("Last heartbeat sent · \(ageText)", .blue) + case "ok-empty", "ok-token": + return ("Heartbeat ok · \(ageText)", .green) + case "skipped": + return ("Heartbeat skipped · \(ageText)", .secondary) + case "failed": + return ("Heartbeat failed · \(ageText)", .red) + default: + return ("Heartbeat · \(ageText)", .secondary) + } + } else { + return ("No heartbeat yet", .secondary) + } + } + + @ViewBuilder + private func statusLine(label: String, color: Color) -> some View { + HStack(spacing: 6) { + Circle() + .fill(color) + .frame(width: 6, height: 6) + Text(label) + .font(.caption) + .foregroundStyle(.secondary) + .multilineTextAlignment(.leading) + .lineLimit(nil) + .fixedSize(horizontal: false, vertical: true) + .layoutPriority(1) + } + .padding(.top, 2) + } + + private var activeBinding: Binding { + Binding(get: { !self.state.isPaused }, set: { self.state.isPaused = !$0 }) + } + + private var heartbeatsBinding: Binding { + Binding(get: { self.state.heartbeatsEnabled }, set: { self.state.heartbeatsEnabled = $0 }) + } + + private var voiceWakeBinding: Binding { + Binding( + get: { self.state.swabbleEnabled }, + set: { newValue in + Task { await self.state.setVoiceWakeEnabled(newValue) } + }) + } + + private var showVoiceWakeMicPicker: Bool { + voiceWakeSupported && self.state.swabbleEnabled + } + + private var voiceWakeMicMenu: some View { + Menu { + self.microphoneMenuItems + + if self.loadingMics { + Divider() + Label("Refreshing microphones…", systemImage: "arrow.triangle.2.circlepath") + .labelStyle(.titleOnly) + .foregroundStyle(.secondary) + .disabled(true) + } + } label: { + HStack { + Text("Microphone") + Spacer() + Text(self.selectedMicLabel) + .foregroundStyle(.secondary) + } + } + .task { await self.loadMicrophones() } + } + + private var selectedMicLabel: String { + if self.state.voiceWakeMicID.isEmpty { return self.defaultMicLabel } + if let match = self.availableMics.first(where: { $0.uid == self.state.voiceWakeMicID }) { + return match.name + } + if !self.state.voiceWakeMicName.isEmpty { return self.state.voiceWakeMicName } + return "Unavailable" + } + + private var microphoneMenuItems: some View { + Group { + if self.isSelectedMicUnavailable { + Label("Disconnected (using System default)", systemImage: "exclamationmark.triangle") + .labelStyle(.titleAndIcon) + .foregroundStyle(.secondary) + .disabled(true) + Divider() + } + Button { + self.state.voiceWakeMicID = "" + self.state.voiceWakeMicName = "" + } label: { + Label(self.defaultMicLabel, systemImage: self.state.voiceWakeMicID.isEmpty ? "checkmark" : "") + .labelStyle(.titleAndIcon) + } + .buttonStyle(.plain) + + ForEach(self.availableMics) { mic in + Button { + self.state.voiceWakeMicID = mic.uid + self.state.voiceWakeMicName = mic.name + } label: { + Label(mic.name, systemImage: self.state.voiceWakeMicID == mic.uid ? "checkmark" : "") + .labelStyle(.titleAndIcon) + } + .buttonStyle(.plain) + } + } + } + + private var isSelectedMicUnavailable: Bool { + let selected = self.state.voiceWakeMicID + guard !selected.isEmpty else { return false } + return !self.availableMics.contains(where: { $0.uid == selected }) + } + + private var defaultMicLabel: String { + if let host = Host.current().localizedName, !host.isEmpty { + return "Auto-detect (\(host))" + } + return "System default" + } + + @MainActor + private func presentDebugResult(_ result: Result, title: String) { + let alert = NSAlert() + alert.messageText = title + switch result { + case let .success(message): + alert.informativeText = message + alert.alertStyle = .informational + case let .failure(error): + alert.informativeText = error.localizedDescription + alert.alertStyle = .warning + } + alert.runModal() + } + + @MainActor + private func loadMicrophones(force: Bool = false) async { + guard self.showVoiceWakeMicPicker else { + self.availableMics = [] + self.loadingMics = false + return + } + if !force, !self.availableMics.isEmpty { return } + self.loadingMics = true + let discovery = AVCaptureDevice.DiscoverySession( + deviceTypes: [.external, .microphone], + mediaType: .audio, + position: .unspecified) + let connectedDevices = discovery.devices.filter(\.isConnected) + self.availableMics = connectedDevices + .sorted { lhs, rhs in + lhs.localizedName.localizedCaseInsensitiveCompare(rhs.localizedName) == .orderedAscending + } + .map { AudioInputDevice(uid: $0.uniqueID, name: $0.localizedName) } + self.availableMics = self.filterAliveInputs(self.availableMics) + self.updateSelectedMicName() + self.loadingMics = false + } + + private func startMicObserver() { + self.micObserver.start { + Task { @MainActor in + self.scheduleMicRefresh() + } + } + } + + @MainActor + private func scheduleMicRefresh() { + self.micRefreshTask?.cancel() + self.micRefreshTask = Task { @MainActor in + try? await Task.sleep(nanoseconds: 300_000_000) + guard !Task.isCancelled else { return } + await self.loadMicrophones(force: true) + } + } + + private func filterAliveInputs(_ inputs: [AudioInputDevice]) -> [AudioInputDevice] { + let aliveUIDs = AudioInputDeviceObserver.aliveInputDeviceUIDs() + guard !aliveUIDs.isEmpty else { return inputs } + return inputs.filter { aliveUIDs.contains($0.uid) } + } + + @MainActor + private func updateSelectedMicName() { + let selected = self.state.voiceWakeMicID + if selected.isEmpty { + self.state.voiceWakeMicName = "" + return + } + if let match = self.availableMics.first(where: { $0.uid == selected }) { + self.state.voiceWakeMicName = match.name + } + } + + private struct AudioInputDevice: Identifiable, Equatable { + let uid: String + let name: String + var id: String { self.uid } + } +} diff --git a/apps/macos/Sources/OpenClaw/MenuContextCardInjector.swift b/apps/macos/Sources/OpenClaw/MenuContextCardInjector.swift new file mode 100644 index 0000000000000000000000000000000000000000..f469ca348dc4a8155529bbb8aad6d09ef23c6558 --- /dev/null +++ b/apps/macos/Sources/OpenClaw/MenuContextCardInjector.swift @@ -0,0 +1,228 @@ +import AppKit +import SwiftUI + +@MainActor +final class MenuContextCardInjector: NSObject, NSMenuDelegate { + static let shared = MenuContextCardInjector() + + private let tag = 9_415_227 + private let fallbackCardWidth: CGFloat = 320 + private var lastKnownMenuWidth: CGFloat? + private weak var originalDelegate: NSMenuDelegate? + private var loadTask: Task? + private var warmTask: Task? + private var cachedRows: [SessionRow] = [] + private var cacheErrorText: String? + private var cacheUpdatedAt: Date? + private let activeWindowSeconds: TimeInterval = 24 * 60 * 60 + private let refreshIntervalSeconds: TimeInterval = 15 + private var isMenuOpen = false + + func install(into statusItem: NSStatusItem) { + // SwiftUI owns the menu, but we can inject a custom NSMenuItem.view right before display. + guard let menu = statusItem.menu else { return } + // Preserve SwiftUI's internal NSMenuDelegate, otherwise it may stop populating menu items. + if menu.delegate !== self { + self.originalDelegate = menu.delegate + menu.delegate = self + } + + if self.warmTask == nil { + self.warmTask = Task { await self.refreshCache(force: true) } + } + } + + func menuWillOpen(_ menu: NSMenu) { + self.originalDelegate?.menuWillOpen?(menu) + self.isMenuOpen = true + + // Remove any previous injected card items. + for item in menu.items where item.tag == self.tag { + menu.removeItem(item) + } + + guard let insertIndex = self.findInsertIndex(in: menu) else { return } + + self.loadTask?.cancel() + + let initialRows = self.cachedRows + let initialIsLoading = initialRows.isEmpty + let initialStatusText = initialIsLoading ? self.cacheErrorText : nil + let initialWidth = self.initialCardWidth(for: menu) + + let initial = AnyView(ContextMenuCardView( + rows: initialRows, + statusText: initialStatusText, + isLoading: initialIsLoading)) + + let hosting = NSHostingView(rootView: initial) + hosting.frame.size.width = max(1, initialWidth) + let size = hosting.fittingSize + hosting.frame = NSRect( + origin: .zero, + size: NSSize(width: initialWidth, height: size.height)) + + let item = NSMenuItem() + item.tag = self.tag + item.view = hosting + item.isEnabled = false + + menu.insertItem(item, at: insertIndex) + + // Capture the menu window width for next open, but do not mutate widths while the menu is visible. + DispatchQueue.main.async { [weak self, weak hosting] in + guard let self, let hosting else { return } + self.captureMenuWidthIfAvailable(for: menu, hosting: hosting) + } + + if initialIsLoading { + self.loadTask = Task { [weak hosting] in + await self.refreshCache(force: true) + guard let hosting else { return } + let view = self.cachedView() + await MainActor.run { + hosting.rootView = view + hosting.invalidateIntrinsicContentSize() + self.captureMenuWidthIfAvailable(for: menu, hosting: hosting) + hosting.frame.size.width = max(1, initialWidth) + let size = hosting.fittingSize + hosting.frame.size.height = size.height + } + } + } else { + // Keep the menu stable while it's open; refresh in the background for next open. + self.loadTask = Task { await self.refreshCache(force: false) } + } + } + + func menuDidClose(_ menu: NSMenu) { + self.originalDelegate?.menuDidClose?(menu) + self.isMenuOpen = false + self.loadTask?.cancel() + } + + func menuNeedsUpdate(_ menu: NSMenu) { + self.originalDelegate?.menuNeedsUpdate?(menu) + } + + func confinementRect(for menu: NSMenu, on screen: NSScreen?) -> NSRect { + if let rect = self.originalDelegate?.confinementRect?(for: menu, on: screen) { + return rect + } + return NSRect.zero + } + + private func refreshCache(force: Bool) async { + if !force, let cacheUpdatedAt, Date().timeIntervalSince(cacheUpdatedAt) < self.refreshIntervalSeconds { + return + } + + do { + let rows = try await self.loadCurrentRows() + self.cachedRows = rows + self.cacheErrorText = nil + self.cacheUpdatedAt = Date() + } catch { + if self.cachedRows.isEmpty { + let raw = (error as? LocalizedError)?.errorDescription ?? error.localizedDescription + let trimmed = raw.trimmingCharacters(in: .whitespacesAndNewlines) + if trimmed.isEmpty { + self.cacheErrorText = "Could not load sessions" + } else { + // Keep the menu readable: one line, short. + let firstLine = trimmed.split(whereSeparator: \.isNewline).first.map(String.init) ?? trimmed + self.cacheErrorText = firstLine.count > 90 ? "\(firstLine.prefix(87))…" : firstLine + } + } + self.cacheUpdatedAt = Date() + } + } + + private func cachedView() -> AnyView { + let rows = self.cachedRows + let isLoading = rows.isEmpty && self.cacheErrorText == nil + return AnyView(ContextMenuCardView(rows: rows, statusText: self.cacheErrorText, isLoading: isLoading)) + } + + private func loadCurrentRows() async throws -> [SessionRow] { + let snapshot = try await SessionLoader.loadSnapshot() + let loaded = snapshot.rows + let now = Date() + let current = loaded.filter { row in + if row.key == "main" { return true } + guard let updatedAt = row.updatedAt else { return false } + return now.timeIntervalSince(updatedAt) <= self.activeWindowSeconds + } + + return current.sorted { lhs, rhs in + if lhs.key == "main" { return true } + if rhs.key == "main" { return false } + return (lhs.updatedAt ?? .distantPast) > (rhs.updatedAt ?? .distantPast) + } + } + + private func findInsertIndex(in menu: NSMenu) -> Int? { + // Prefer inserting before the first separator (so the card sits right below the Active toggle). + if let idx = menu.items.firstIndex(where: { $0.title == "Send Heartbeats" }) { + // SwiftUI menus typically include a separator right after the first toggle; insert before it so the + // separator appears below the context card. + if let sepIdx = menu.items[..= 1 { return 1 } + return menu.items.count + } + + private func initialCardWidth(for menu: NSMenu) -> CGFloat { + let widthCandidates: [CGFloat] = [ + menu.minimumWidth, + self.lastKnownMenuWidth ?? 0, + self.fallbackCardWidth, + ] + let resolved = widthCandidates.max() ?? self.fallbackCardWidth + return max(300, resolved) + } + + private func captureMenuWidthIfAvailable(for menu: NSMenu, hosting: NSHostingView) { + let targetWidth: CGFloat? = { + if let contentWidth = hosting.window?.contentView?.bounds.width, contentWidth > 0 { return contentWidth } + if let superWidth = hosting.superview?.bounds.width, superWidth > 0 { return superWidth } + let minimumWidth = menu.minimumWidth + if minimumWidth > 0 { return minimumWidth } + return nil + }() + + guard let targetWidth else { return } + self.lastKnownMenuWidth = max(300, targetWidth) + } +} + +#if DEBUG +extension MenuContextCardInjector { + func _testSetCache(rows: [SessionRow], errorText: String?, updatedAt: Date?) { + self.cachedRows = rows + self.cacheErrorText = errorText + self.cacheUpdatedAt = updatedAt + } + + func _testFindInsertIndex(in menu: NSMenu) -> Int? { + self.findInsertIndex(in: menu) + } + + func _testInitialCardWidth(for menu: NSMenu) -> CGFloat { + self.initialCardWidth(for: menu) + } + + func _testCachedView() -> AnyView { + self.cachedView() + } +} +#endif diff --git a/apps/macos/Sources/OpenClaw/MenuHighlightedHostView.swift b/apps/macos/Sources/OpenClaw/MenuHighlightedHostView.swift new file mode 100644 index 0000000000000000000000000000000000000000..f1e85cba1528ff1ba2e1da74e2f08b3f3d94ab04 --- /dev/null +++ b/apps/macos/Sources/OpenClaw/MenuHighlightedHostView.swift @@ -0,0 +1,102 @@ +import AppKit +import SwiftUI + +final class HighlightedMenuItemHostView: NSView { + private var baseView: AnyView + private let hosting: NSHostingView + private var targetWidth: CGFloat + private var tracking: NSTrackingArea? + private var hovered = false { + didSet { self.updateHighlight() } + } + + init(rootView: AnyView, width: CGFloat) { + self.baseView = rootView + self.hosting = NSHostingView(rootView: AnyView(rootView.environment(\.menuItemHighlighted, false))) + self.targetWidth = max(1, width) + super.init(frame: .zero) + + self.addSubview(self.hosting) + self.hosting.autoresizingMask = [.width, .height] + self.updateSizing() + } + + @available(*, unavailable) + required init?(coder: NSCoder) { fatalError("init(coder:) has not been implemented") } + + override var intrinsicContentSize: NSSize { + let size = self.hosting.fittingSize + return NSSize(width: self.targetWidth, height: size.height) + } + + override func updateTrackingAreas() { + super.updateTrackingAreas() + if let tracking { + self.removeTrackingArea(tracking) + } + let options: NSTrackingArea.Options = [ + .mouseEnteredAndExited, + .activeAlways, + .inVisibleRect, + ] + let area = NSTrackingArea(rect: self.bounds, options: options, owner: self, userInfo: nil) + self.addTrackingArea(area) + self.tracking = area + } + + override func mouseEntered(with event: NSEvent) { + _ = event + self.hovered = true + } + + override func mouseExited(with event: NSEvent) { + _ = event + self.hovered = false + } + + override func layout() { + super.layout() + self.hosting.frame = self.bounds + } + + override func draw(_ dirtyRect: NSRect) { + if self.hovered { + NSColor.selectedContentBackgroundColor.setFill() + self.bounds.fill() + } + super.draw(dirtyRect) + } + + func update(rootView: AnyView, width: CGFloat) { + self.baseView = rootView + self.targetWidth = max(1, width) + self.updateHighlight() + } + + private func updateHighlight() { + self.hosting.rootView = AnyView(self.baseView.environment(\.menuItemHighlighted, self.hovered)) + self.updateSizing() + self.needsDisplay = true + } + + private func updateSizing() { + let width = max(1, self.targetWidth) + self.hosting.frame.size.width = width + let size = self.hosting.fittingSize + self.frame = NSRect(origin: .zero, size: NSSize(width: width, height: size.height)) + self.invalidateIntrinsicContentSize() + } +} + +struct MenuHostedHighlightedItem: NSViewRepresentable { + let width: CGFloat + let rootView: AnyView + + func makeNSView(context _: Context) -> HighlightedMenuItemHostView { + HighlightedMenuItemHostView(rootView: self.rootView, width: self.width) + } + + func updateNSView(_ nsView: HighlightedMenuItemHostView, context _: Context) { + nsView.update(rootView: self.rootView, width: self.width) + } +} diff --git a/apps/macos/Sources/OpenClaw/MenuHostedItem.swift b/apps/macos/Sources/OpenClaw/MenuHostedItem.swift new file mode 100644 index 0000000000000000000000000000000000000000..c5a2b73cd94702cc210b0d03eca38f367280d826 --- /dev/null +++ b/apps/macos/Sources/OpenClaw/MenuHostedItem.swift @@ -0,0 +1,29 @@ +import AppKit +import SwiftUI + +/// Hosts arbitrary SwiftUI content as an AppKit view so it can be embedded in a native `NSMenuItem.view`. +/// +/// SwiftUI `MenuBarExtraStyle.menu` aggressively simplifies many view hierarchies into a title + image. +/// Wrapping the content in an `NSViewRepresentable` forces AppKit-backed menu item rendering. +struct MenuHostedItem: NSViewRepresentable { + let width: CGFloat + let rootView: AnyView + + func makeNSView(context _: Context) -> NSHostingView { + let hosting = NSHostingView(rootView: self.rootView) + self.applySizing(to: hosting) + return hosting + } + + func updateNSView(_ nsView: NSHostingView, context _: Context) { + nsView.rootView = self.rootView + self.applySizing(to: nsView) + } + + private func applySizing(to hosting: NSHostingView) { + let width = max(1, self.width) + hosting.frame.size.width = width + let fitting = hosting.fittingSize + hosting.frame = NSRect(origin: .zero, size: NSSize(width: width, height: fitting.height)) + } +} diff --git a/apps/macos/Sources/OpenClaw/MenuSessionsHeaderView.swift b/apps/macos/Sources/OpenClaw/MenuSessionsHeaderView.swift new file mode 100644 index 0000000000000000000000000000000000000000..e96cea53b8437f77dc6b7ea62563385f84c940b1 --- /dev/null +++ b/apps/macos/Sources/OpenClaw/MenuSessionsHeaderView.swift @@ -0,0 +1,44 @@ +import SwiftUI + +struct MenuSessionsHeaderView: View { + let count: Int + let statusText: String? + + private let paddingTop: CGFloat = 8 + private let paddingBottom: CGFloat = 6 + private let paddingTrailing: CGFloat = 10 + private let paddingLeading: CGFloat = 20 + + var body: some View { + VStack(alignment: .leading, spacing: 6) { + HStack(alignment: .firstTextBaseline) { + Text("Context") + .font(.caption.weight(.semibold)) + .foregroundStyle(.secondary) + Spacer(minLength: 10) + Text(self.subtitle) + .font(.caption) + .foregroundStyle(.secondary) + } + + if let statusText, !statusText.isEmpty { + Text(statusText) + .font(.caption) + .foregroundStyle(.secondary) + .lineLimit(1) + .truncationMode(.tail) + } + } + .padding(.top, self.paddingTop) + .padding(.bottom, self.paddingBottom) + .padding(.leading, self.paddingLeading) + .padding(.trailing, self.paddingTrailing) + .frame(minWidth: 300, maxWidth: .infinity, alignment: .leading) + .transaction { txn in txn.animation = nil } + } + + private var subtitle: String { + if self.count == 1 { return "1 session · 24h" } + return "\(self.count) sessions · 24h" + } +} diff --git a/apps/macos/Sources/OpenClaw/MenuSessionsInjector.swift b/apps/macos/Sources/OpenClaw/MenuSessionsInjector.swift new file mode 100644 index 0000000000000000000000000000000000000000..7ab9a64ca62bfea8b2514f27dcd62f567a8dce24 --- /dev/null +++ b/apps/macos/Sources/OpenClaw/MenuSessionsInjector.swift @@ -0,0 +1,1228 @@ +import AppKit +import Foundation +import Observation +import SwiftUI + +@MainActor +final class MenuSessionsInjector: NSObject, NSMenuDelegate { + static let shared = MenuSessionsInjector() + + private let tag = 9_415_557 + private let nodesTag = 9_415_558 + private let fallbackWidth: CGFloat = 320 + private let activeWindowSeconds: TimeInterval = 24 * 60 * 60 + + private weak var originalDelegate: NSMenuDelegate? + private weak var statusItem: NSStatusItem? + private var loadTask: Task? + private var nodesLoadTask: Task? + private var previewTasks: [Task] = [] + private var isMenuOpen = false + private var lastKnownMenuWidth: CGFloat? + private var menuOpenWidth: CGFloat? + private var isObservingControlChannel = false + + private var cachedSnapshot: SessionStoreSnapshot? + private var cachedErrorText: String? + private var cacheUpdatedAt: Date? + private let refreshIntervalSeconds: TimeInterval = 12 + private var cachedUsageSummary: GatewayUsageSummary? + private var cachedUsageErrorText: String? + private var usageCacheUpdatedAt: Date? + private let usageRefreshIntervalSeconds: TimeInterval = 30 + private var cachedCostSummary: GatewayCostUsageSummary? + private var cachedCostErrorText: String? + private var costCacheUpdatedAt: Date? + private let costRefreshIntervalSeconds: TimeInterval = 45 + private let nodesStore = NodesStore.shared + #if DEBUG + private var testControlChannelConnected: Bool? + #endif + + func install(into statusItem: NSStatusItem) { + self.statusItem = statusItem + guard let menu = statusItem.menu else { return } + + // Preserve SwiftUI's internal NSMenuDelegate, otherwise it may stop populating menu items. + if menu.delegate !== self { + self.originalDelegate = menu.delegate + menu.delegate = self + } + + if self.loadTask == nil { + self.loadTask = Task { await self.refreshCache(force: true) } + } + + self.startControlChannelObservation() + self.nodesStore.start() + } + + func menuWillOpen(_ menu: NSMenu) { + self.originalDelegate?.menuWillOpen?(menu) + self.isMenuOpen = true + self.menuOpenWidth = self.currentMenuWidth(for: menu) + + self.inject(into: menu) + self.injectNodes(into: menu) + + // Refresh in background for the next open; keep width stable while open. + self.loadTask?.cancel() + let forceRefresh = self.cachedSnapshot == nil || self.cachedErrorText != nil + self.loadTask = Task { [weak self] in + guard let self else { return } + await self.refreshCache(force: forceRefresh) + await self.refreshUsageCache(force: forceRefresh) + await self.refreshCostUsageCache(force: forceRefresh) + await MainActor.run { + guard self.isMenuOpen else { return } + self.inject(into: menu) + self.injectNodes(into: menu) + } + } + + self.nodesLoadTask?.cancel() + self.nodesLoadTask = Task { [weak self] in + guard let self else { return } + await self.nodesStore.refresh() + await MainActor.run { + guard self.isMenuOpen else { return } + self.injectNodes(into: menu) + } + } + } + + func menuDidClose(_ menu: NSMenu) { + self.originalDelegate?.menuDidClose?(menu) + self.isMenuOpen = false + self.menuOpenWidth = nil + self.loadTask?.cancel() + self.nodesLoadTask?.cancel() + self.cancelPreviewTasks() + } + + private func startControlChannelObservation() { + guard !self.isObservingControlChannel else { return } + self.isObservingControlChannel = true + self.observeControlChannelState() + } + + private func observeControlChannelState() { + withObservationTracking { + _ = ControlChannel.shared.state + } onChange: { [weak self] in + Task { @MainActor [weak self] in + guard let self else { return } + self.handleControlChannelStateChange() + self.observeControlChannelState() + } + } + } + + private func handleControlChannelStateChange() { + guard self.isMenuOpen, let menu = self.statusItem?.menu else { return } + self.loadTask?.cancel() + self.loadTask = Task { [weak self, weak menu] in + guard let self, let menu else { return } + await self.refreshCache(force: true) + await self.refreshUsageCache(force: true) + await self.refreshCostUsageCache(force: true) + await MainActor.run { + guard self.isMenuOpen else { return } + self.inject(into: menu) + self.injectNodes(into: menu) + } + } + + self.nodesLoadTask?.cancel() + self.nodesLoadTask = Task { [weak self, weak menu] in + guard let self, let menu else { return } + await self.nodesStore.refresh() + await MainActor.run { + guard self.isMenuOpen else { return } + self.injectNodes(into: menu) + } + } + } + + func menuNeedsUpdate(_ menu: NSMenu) { + self.originalDelegate?.menuNeedsUpdate?(menu) + } + + func confinementRect(for menu: NSMenu, on screen: NSScreen?) -> NSRect { + if let rect = self.originalDelegate?.confinementRect?(for: menu, on: screen) { + return rect + } + return NSRect.zero + } +} + +extension MenuSessionsInjector { + // MARK: - Injection + + private var mainSessionKey: String { WorkActivityStore.shared.mainSessionKey } + + private func inject(into menu: NSMenu) { + self.cancelPreviewTasks() + // Remove any previous injected items. + for item in menu.items where item.tag == self.tag { + menu.removeItem(item) + } + + guard let insertIndex = self.findInsertIndex(in: menu) else { return } + let width = self.initialWidth(for: menu) + let isConnected = self.isControlChannelConnected + let channelState = ControlChannel.shared.state + + var cursor = insertIndex + var headerView: NSView? + + if let snapshot = self.cachedSnapshot { + let now = Date() + let mainKey = self.mainSessionKey + let rows = snapshot.rows.filter { row in + if row.key == "main", mainKey != "main" { return false } + if row.key == mainKey { return true } + guard let updatedAt = row.updatedAt else { return false } + return now.timeIntervalSince(updatedAt) <= self.activeWindowSeconds + }.sorted { lhs, rhs in + if lhs.key == mainKey { return true } + if rhs.key == mainKey { return false } + return (lhs.updatedAt ?? .distantPast) > (rhs.updatedAt ?? .distantPast) + } + if !rows.isEmpty { + let previewKeys = rows.prefix(20).map(\.key) + let task = Task { + await SessionMenuPreviewLoader.prewarm(sessionKeys: previewKeys, maxItems: 10) + } + self.previewTasks.append(task) + } + + let headerItem = NSMenuItem() + headerItem.tag = self.tag + headerItem.isEnabled = false + let statusText = self + .cachedErrorText ?? (isConnected ? nil : self.controlChannelStatusText(for: channelState)) + let hosted = self.makeHostedView( + rootView: AnyView(MenuSessionsHeaderView( + count: rows.count, + statusText: statusText)), + width: width, + highlighted: false) + headerItem.view = hosted + headerView = hosted + menu.insertItem(headerItem, at: cursor) + cursor += 1 + + if rows.isEmpty { + menu.insertItem( + self.makeMessageItem(text: "No active sessions", symbolName: "minus", width: width), + at: cursor) + cursor += 1 + } else { + for row in rows { + let item = NSMenuItem() + item.tag = self.tag + item.isEnabled = true + item.submenu = self.buildSubmenu(for: row, storePath: snapshot.storePath) + item.view = self.makeHostedView( + rootView: AnyView(SessionMenuLabelView(row: row, width: width)), + width: width, + highlighted: true) + menu.insertItem(item, at: cursor) + cursor += 1 + } + } + } else { + let headerItem = NSMenuItem() + headerItem.tag = self.tag + headerItem.isEnabled = false + let statusText = isConnected + ? (self.cachedErrorText ?? "Loading sessions…") + : self.controlChannelStatusText(for: channelState) + let hosted = self.makeHostedView( + rootView: AnyView(MenuSessionsHeaderView( + count: 0, + statusText: statusText)), + width: width, + highlighted: false) + headerItem.view = hosted + headerView = hosted + menu.insertItem(headerItem, at: cursor) + cursor += 1 + + if !isConnected { + menu.insertItem( + self.makeMessageItem( + text: "Connect the gateway to see sessions", + symbolName: "bolt.slash", + width: width), + at: cursor) + cursor += 1 + } + } + + cursor = self.insertUsageSection(into: menu, at: cursor, width: width) + cursor = self.insertCostUsageSection(into: menu, at: cursor, width: width) + + DispatchQueue.main.async { [weak self, weak headerView] in + guard let self, let headerView else { return } + self.captureMenuWidthIfAvailable(from: headerView) + } + } + + private func injectNodes(into menu: NSMenu) { + for item in menu.items where item.tag == self.nodesTag { + menu.removeItem(item) + } + + guard let insertIndex = self.findNodesInsertIndex(in: menu) else { return } + let width = self.initialWidth(for: menu) + var cursor = insertIndex + + let entries = self.sortedNodeEntries() + let topSeparator = NSMenuItem.separator() + topSeparator.tag = self.nodesTag + menu.insertItem(topSeparator, at: cursor) + cursor += 1 + + if let gatewayEntry = self.gatewayEntry() { + let gatewayItem = self.makeNodeItem(entry: gatewayEntry, width: width) + menu.insertItem(gatewayItem, at: cursor) + cursor += 1 + } + + if case .connecting = ControlChannel.shared.state { + menu.insertItem( + self.makeMessageItem(text: "Connecting…", symbolName: "circle.dashed", width: width), + at: cursor) + cursor += 1 + return + } + + guard self.isControlChannelConnected else { return } + + if let error = self.nodesStore.lastError?.nonEmpty { + menu.insertItem( + self.makeMessageItem( + text: "Error: \(error)", + symbolName: "exclamationmark.triangle", + width: width), + at: cursor) + cursor += 1 + } else if let status = self.nodesStore.statusMessage?.nonEmpty { + menu.insertItem( + self.makeMessageItem(text: status, symbolName: "info.circle", width: width), + at: cursor) + cursor += 1 + } + + if entries.isEmpty { + let title = self.nodesStore.isLoading ? "Loading devices..." : "No devices yet" + menu.insertItem( + self.makeMessageItem(text: title, symbolName: "circle.dashed", width: width), + at: cursor) + cursor += 1 + } else { + for entry in entries.prefix(8) { + let item = self.makeNodeItem(entry: entry, width: width) + menu.insertItem(item, at: cursor) + cursor += 1 + } + + if entries.count > 8 { + let moreItem = NSMenuItem() + moreItem.tag = self.nodesTag + moreItem.title = "More Devices..." + moreItem.image = NSImage(systemSymbolName: "ellipsis.circle", accessibilityDescription: nil) + let overflow = Array(entries.dropFirst(8)) + moreItem.submenu = self.buildNodesOverflowMenu(entries: overflow, width: width) + menu.insertItem(moreItem, at: cursor) + cursor += 1 + } + } + + _ = cursor + } + + private func insertUsageSection(into menu: NSMenu, at cursor: Int, width: CGFloat) -> Int { + let rows = self.usageRows + if rows.isEmpty { + return cursor + } + + var cursor = cursor + + if cursor > 0, !menu.items[cursor - 1].isSeparatorItem { + let separator = NSMenuItem.separator() + separator.tag = self.tag + menu.insertItem(separator, at: cursor) + cursor += 1 + } + + let headerItem = NSMenuItem() + headerItem.tag = self.tag + headerItem.isEnabled = false + headerItem.view = self.makeHostedView( + rootView: AnyView(MenuUsageHeaderView( + count: rows.count)), + width: width, + highlighted: false) + menu.insertItem(headerItem, at: cursor) + cursor += 1 + + if let selectedProvider = self.selectedUsageProviderId, + let primary = rows.first(where: { $0.providerId.lowercased() == selectedProvider }), + rows.count > 1 + { + let others = rows.filter { $0.providerId.lowercased() != selectedProvider } + + let item = NSMenuItem() + item.tag = self.tag + item.isEnabled = true + if !others.isEmpty { + item.submenu = self.buildUsageOverflowMenu(rows: others, width: width) + } + item.view = self.makeHostedView( + rootView: AnyView(UsageMenuLabelView(row: primary, width: width, showsChevron: !others.isEmpty)), + width: width, + highlighted: true) + menu.insertItem(item, at: cursor) + cursor += 1 + + return cursor + } + + for row in rows { + let item = NSMenuItem() + item.tag = self.tag + item.isEnabled = false + item.view = self.makeHostedView( + rootView: AnyView(UsageMenuLabelView(row: row, width: width)), + width: width, + highlighted: false) + menu.insertItem(item, at: cursor) + cursor += 1 + } + + return cursor + } + + private func insertCostUsageSection(into menu: NSMenu, at cursor: Int, width: CGFloat) -> Int { + guard self.isControlChannelConnected else { return cursor } + guard let submenu = self.buildCostUsageSubmenu(width: width) else { return cursor } + var cursor = cursor + + if cursor > 0, !menu.items[cursor - 1].isSeparatorItem { + let separator = NSMenuItem.separator() + separator.tag = self.tag + menu.insertItem(separator, at: cursor) + cursor += 1 + } + + let item = NSMenuItem(title: "Usage cost (30 days)", action: nil, keyEquivalent: "") + item.tag = self.tag + item.isEnabled = true + item.image = NSImage(systemSymbolName: "chart.bar.xaxis", accessibilityDescription: nil) + item.submenu = submenu + menu.insertItem(item, at: cursor) + cursor += 1 + return cursor + } + + private var selectedUsageProviderId: String? { + guard let model = self.cachedSnapshot?.defaults.model.nonEmpty else { return nil } + let trimmed = model.trimmingCharacters(in: .whitespacesAndNewlines) + guard let slash = trimmed.firstIndex(of: "/") else { return nil } + let provider = trimmed[.. NSMenu { + let menu = NSMenu() + for row in rows { + let item = NSMenuItem() + item.tag = self.tag + item.isEnabled = false + item.view = self.makeHostedView( + rootView: AnyView(UsageMenuLabelView(row: row, width: width)), + width: width, + highlighted: false) + menu.addItem(item) + } + return menu + } + + private var isControlChannelConnected: Bool { + #if DEBUG + if let override = self.testControlChannelConnected { return override } + #endif + if case .connected = ControlChannel.shared.state { return true } + return false + } + + private func controlChannelStatusText(for state: ControlChannel.ConnectionState) -> String { + switch state { + case .connected: + "Loading sessions…" + case .connecting: + "Connecting…" + case let .degraded(message): + message.nonEmpty ?? "Gateway disconnected" + case .disconnected: + "Gateway disconnected" + } + } + + private func buildCostUsageSubmenu(width: CGFloat) -> NSMenu? { + if let error = self.cachedCostErrorText, !error.isEmpty, self.cachedCostSummary == nil { + let menu = NSMenu() + let item = NSMenuItem(title: error, action: nil, keyEquivalent: "") + item.isEnabled = false + menu.addItem(item) + return menu + } + + guard let summary = self.cachedCostSummary else { return nil } + guard !summary.daily.isEmpty else { return nil } + + let menu = NSMenu() + menu.delegate = self + + let chartView = CostUsageHistoryMenuView(summary: summary, width: width) + let hosting = NSHostingView(rootView: AnyView(chartView)) + let controller = NSHostingController(rootView: AnyView(chartView)) + let size = controller.sizeThatFits(in: CGSize(width: width, height: .greatestFiniteMagnitude)) + hosting.frame = NSRect(origin: .zero, size: NSSize(width: width, height: size.height)) + + let chartItem = NSMenuItem() + chartItem.view = hosting + chartItem.isEnabled = false + chartItem.representedObject = "costUsageChart" + menu.addItem(chartItem) + + return menu + } + + private func gatewayEntry() -> NodeInfo? { + let mode = AppStateStore.shared.connectionMode + let isConnected = self.isControlChannelConnected + let port = GatewayEnvironment.gatewayPort() + var host: String? + var platform: String? + + switch mode { + case .remote: + platform = "remote" + if AppStateStore.shared.remoteTransport == .direct { + let trimmedUrl = AppStateStore.shared.remoteUrl + .trimmingCharacters(in: .whitespacesAndNewlines) + if let url = URL(string: trimmedUrl), let urlHost = url.host, !urlHost.isEmpty { + if let port = url.port { + host = "\(urlHost):\(port)" + } else { + host = urlHost + } + } else { + host = trimmedUrl.nonEmpty + } + } else { + let target = AppStateStore.shared.remoteTarget + if let parsed = CommandResolver.parseSSHTarget(target) { + host = parsed.port == 22 ? parsed.host : "\(parsed.host):\(parsed.port)" + } else { + host = target.nonEmpty + } + } + case .local: + platform = "local" + host = GatewayConnectivityCoordinator.shared.localEndpointHostLabel ?? "127.0.0.1:\(port)" + case .unconfigured: + platform = nil + host = nil + } + + return NodeInfo( + nodeId: "gateway", + displayName: "Gateway", + platform: platform, + version: nil, + coreVersion: nil, + uiVersion: nil, + deviceFamily: nil, + modelIdentifier: nil, + remoteIp: host, + caps: nil, + commands: nil, + permissions: nil, + paired: nil, + connected: isConnected) + } + + private func makeNodeItem(entry: NodeInfo, width: CGFloat) -> NSMenuItem { + let item = NSMenuItem() + item.tag = self.nodesTag + item.target = self + item.action = #selector(self.copyNodeSummary(_:)) + item.representedObject = NodeMenuEntryFormatter.summaryText(entry) + item.view = HighlightedMenuItemHostView( + rootView: AnyView(NodeMenuRowView(entry: entry, width: width)), + width: width) + item.submenu = self.buildNodeSubmenu(entry: entry, width: width) + return item + } + + private func makeSessionPreviewItem( + sessionKey: String, + title: String, + width: CGFloat, + maxLines: Int) -> NSMenuItem + { + let item = NSMenuItem() + item.tag = self.tag + item.isEnabled = false + let view = AnyView(SessionMenuPreviewView( + width: width, + maxLines: maxLines, + title: title, + items: [], + status: .loading)) + let hosting = NSHostingView(rootView: view) + hosting.frame.size.width = max(1, width) + let size = hosting.fittingSize + hosting.frame = NSRect(origin: .zero, size: NSSize(width: width, height: size.height)) + item.view = hosting + + let task = Task { [weak hosting] in + let snapshot = await SessionMenuPreviewLoader.load(sessionKey: sessionKey, maxItems: 10) + guard !Task.isCancelled else { return } + await MainActor.run { + guard let hosting else { return } + let nextView = AnyView(SessionMenuPreviewView( + width: width, + maxLines: maxLines, + title: title, + items: snapshot.items, + status: snapshot.status)) + hosting.rootView = nextView + hosting.invalidateIntrinsicContentSize() + hosting.frame.size.width = max(1, width) + let size = hosting.fittingSize + hosting.frame.size.height = size.height + } + } + self.previewTasks.append(task) + return item + } + + private func cancelPreviewTasks() { + for task in self.previewTasks { + task.cancel() + } + self.previewTasks.removeAll() + } + + private func makeMessageItem(text: String, symbolName: String, width: CGFloat, maxLines: Int? = 2) -> NSMenuItem { + let view = AnyView( + HStack(alignment: .top, spacing: 8) { + Image(systemName: symbolName) + .font(.caption) + .foregroundStyle(.secondary) + .frame(width: 14, alignment: .leading) + .padding(.top, 1) + + Text(text) + .font(.caption) + .foregroundStyle(.secondary) + .multilineTextAlignment(.leading) + .lineLimit(maxLines) + .truncationMode(.tail) + .fixedSize(horizontal: false, vertical: true) + .layoutPriority(1) + .frame(maxWidth: .infinity, alignment: .leading) + + Spacer(minLength: 0) + } + .padding(.leading, 18) + .padding(.trailing, 12) + .padding(.vertical, 6) + .frame(width: max(1, width), alignment: .leading)) + + let item = NSMenuItem() + item.tag = self.tag + item.isEnabled = false + item.view = self.makeHostedView(rootView: view, width: width, highlighted: false) + return item + } +} + +extension MenuSessionsInjector { + // MARK: - Cache + + private func refreshCache(force: Bool) async { + if !force, let updated = self.cacheUpdatedAt, Date().timeIntervalSince(updated) < self.refreshIntervalSeconds { + return + } + + guard self.isControlChannelConnected else { + if self.cachedSnapshot != nil { + self.cachedErrorText = "Gateway disconnected (showing cached)" + } else { + self.cachedErrorText = nil + } + self.cacheUpdatedAt = Date() + return + } + + do { + self.cachedSnapshot = try await SessionLoader.loadSnapshot(limit: 32) + self.cachedErrorText = nil + self.cacheUpdatedAt = Date() + } catch { + self.cachedSnapshot = nil + self.cachedErrorText = self.compactError(error) + self.cacheUpdatedAt = Date() + } + } + + private func refreshUsageCache(force: Bool) async { + if !force, + let updated = self.usageCacheUpdatedAt, + Date().timeIntervalSince(updated) < self.usageRefreshIntervalSeconds + { + return + } + + guard self.isControlChannelConnected else { + self.usageCacheUpdatedAt = Date() + return + } + + do { + self.cachedUsageSummary = try await UsageLoader.loadSummary() + } catch { + self.cachedUsageSummary = nil + self.cachedUsageErrorText = nil + } + self.usageCacheUpdatedAt = Date() + } + + private func refreshCostUsageCache(force: Bool) async { + if !force, + let updated = self.costCacheUpdatedAt, + Date().timeIntervalSince(updated) < self.costRefreshIntervalSeconds + { + return + } + + guard self.isControlChannelConnected else { + self.costCacheUpdatedAt = Date() + return + } + + do { + self.cachedCostSummary = try await CostUsageLoader.loadSummary() + self.cachedCostErrorText = nil + } catch { + self.cachedCostSummary = nil + self.cachedCostErrorText = self.compactUsageError(error) + } + self.costCacheUpdatedAt = Date() + } + + private func compactUsageError(_ error: Error) -> String { + let message = error.localizedDescription.trimmingCharacters(in: .whitespacesAndNewlines) + if message.isEmpty { return "Usage unavailable" } + if message.count > 90 { return "\(message.prefix(87))…" } + return message + } + + private func compactError(_ error: Error) -> String { + if let loadError = error as? SessionLoadError { + switch loadError { + case .gatewayUnavailable: + return "No connection to gateway" + case .decodeFailed: + return "Sessions unavailable" + } + } + return "Sessions unavailable" + } +} + +extension MenuSessionsInjector { + // MARK: - Submenus + + private func buildSubmenu(for row: SessionRow, storePath: String) -> NSMenu { + let menu = NSMenu() + let width = self.submenuWidth() + + menu.addItem(self.makeSessionPreviewItem( + sessionKey: row.key, + title: "Recent messages (last 10)", + width: width, + maxLines: 3)) + + let morePreview = NSMenuItem(title: "More preview…", action: nil, keyEquivalent: "") + morePreview.submenu = self.buildPreviewSubmenu(sessionKey: row.key, width: width) + menu.addItem(morePreview) + + menu.addItem(NSMenuItem.separator()) + + let thinking = NSMenuItem(title: "Thinking", action: nil, keyEquivalent: "") + thinking.submenu = self.buildThinkingMenu(for: row) + menu.addItem(thinking) + + let verbose = NSMenuItem(title: "Verbose", action: nil, keyEquivalent: "") + verbose.submenu = self.buildVerboseMenu(for: row) + menu.addItem(verbose) + + if AppStateStore.shared.debugPaneEnabled, + AppStateStore.shared.connectionMode == .local, + let sessionId = row.sessionId, + !sessionId.isEmpty + { + menu.addItem(NSMenuItem.separator()) + let openLog = NSMenuItem( + title: "Open Session Log", + action: #selector(self.openSessionLog(_:)), + keyEquivalent: "") + openLog.target = self + openLog.representedObject = [ + "sessionId": sessionId, + "storePath": storePath, + ] + menu.addItem(openLog) + } + + menu.addItem(NSMenuItem.separator()) + + let reset = NSMenuItem(title: "Reset Session", action: #selector(self.resetSession(_:)), keyEquivalent: "") + reset.target = self + reset.representedObject = row.key + menu.addItem(reset) + + let compact = NSMenuItem( + title: "Compact Session Log", + action: #selector(self.compactSession(_:)), + keyEquivalent: "") + compact.target = self + compact.representedObject = row.key + menu.addItem(compact) + + if row.key != self.mainSessionKey, row.key != "global" { + let del = NSMenuItem(title: "Delete Session", action: #selector(self.deleteSession(_:)), keyEquivalent: "") + del.target = self + del.representedObject = row.key + del.isAlternate = false + del.keyEquivalentModifierMask = [] + menu.addItem(del) + } + + return menu + } + + private func buildThinkingMenu(for row: SessionRow) -> NSMenu { + let menu = NSMenu() + menu.autoenablesItems = false + menu.showsStateColumn = true + let levels: [String] = ["off", "minimal", "low", "medium", "high"] + let current = levels.contains(row.thinkingLevel ?? "") ? row.thinkingLevel ?? "off" : "off" + for level in levels { + let title = level.capitalized + let item = NSMenuItem(title: title, action: #selector(self.patchThinking(_:)), keyEquivalent: "") + item.target = self + item.representedObject = [ + "key": row.key, + "value": level as Any, + ] + item.state = (current == level) ? .on : .off + menu.addItem(item) + } + return menu + } + + private func buildVerboseMenu(for row: SessionRow) -> NSMenu { + let menu = NSMenu() + menu.autoenablesItems = false + menu.showsStateColumn = true + let levels: [String] = ["on", "off"] + let current = levels.contains(row.verboseLevel ?? "") ? row.verboseLevel ?? "off" : "off" + for level in levels { + let title = level.capitalized + let item = NSMenuItem(title: title, action: #selector(self.patchVerbose(_:)), keyEquivalent: "") + item.target = self + item.representedObject = [ + "key": row.key, + "value": level as Any, + ] + item.state = (current == level) ? .on : .off + menu.addItem(item) + } + return menu + } + + private func buildPreviewSubmenu(sessionKey: String, width: CGFloat) -> NSMenu { + let menu = NSMenu() + menu.addItem(self.makeSessionPreviewItem( + sessionKey: sessionKey, + title: "Recent messages (expanded)", + width: width, + maxLines: 8)) + return menu + } + + private func buildNodesOverflowMenu(entries: [NodeInfo], width: CGFloat) -> NSMenu { + let menu = NSMenu() + for entry in entries { + let item = NSMenuItem() + item.target = self + item.action = #selector(self.copyNodeSummary(_:)) + item.representedObject = NodeMenuEntryFormatter.summaryText(entry) + item.view = HighlightedMenuItemHostView( + rootView: AnyView(NodeMenuRowView(entry: entry, width: width)), + width: width) + item.submenu = self.buildNodeSubmenu(entry: entry, width: width) + menu.addItem(item) + } + return menu + } + + private func buildNodeSubmenu(entry: NodeInfo, width: CGFloat) -> NSMenu { + let menu = NSMenu() + menu.autoenablesItems = false + + menu.addItem(self.makeNodeCopyItem(label: "Node ID", value: entry.nodeId)) + + if let name = entry.displayName?.nonEmpty { + menu.addItem(self.makeNodeCopyItem(label: "Name", value: name)) + } + + if let ip = entry.remoteIp?.nonEmpty { + menu.addItem(self.makeNodeCopyItem(label: "IP", value: ip)) + } + + menu.addItem(self.makeNodeCopyItem(label: "Status", value: NodeMenuEntryFormatter.roleText(entry))) + + if let platform = NodeMenuEntryFormatter.platformText(entry) { + menu.addItem(self.makeNodeCopyItem(label: "Platform", value: platform)) + } + + if let version = NodeMenuEntryFormatter.detailRightVersion(entry)?.nonEmpty { + menu.addItem(self.makeNodeCopyItem(label: "Version", value: version)) + } + + menu.addItem(self.makeNodeDetailItem(label: "Connected", value: entry.isConnected ? "Yes" : "No")) + menu.addItem(self.makeNodeDetailItem(label: "Paired", value: entry.isPaired ? "Yes" : "No")) + + if let caps = entry.caps?.filter({ !$0.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty }), + !caps.isEmpty + { + menu.addItem(self.makeNodeCopyItem(label: "Caps", value: caps.joined(separator: ", "))) + } + + if let commands = entry.commands?.filter({ !$0.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty }), + !commands.isEmpty + { + menu.addItem(self.makeNodeMultilineItem( + label: "Commands", + value: commands.joined(separator: ", "), + width: width)) + } + + return menu + } + + private func makeNodeDetailItem(label: String, value: String) -> NSMenuItem { + let item = NSMenuItem(title: "\(label): \(value)", action: nil, keyEquivalent: "") + item.isEnabled = false + return item + } + + private func makeNodeCopyItem(label: String, value: String) -> NSMenuItem { + let item = NSMenuItem(title: "\(label): \(value)", action: #selector(self.copyNodeValue(_:)), keyEquivalent: "") + item.target = self + item.representedObject = value + return item + } + + private func makeNodeMultilineItem(label: String, value: String, width: CGFloat) -> NSMenuItem { + let item = NSMenuItem() + item.target = self + item.action = #selector(self.copyNodeValue(_:)) + item.representedObject = value + item.view = HighlightedMenuItemHostView( + rootView: AnyView(NodeMenuMultilineView(label: label, value: value, width: width)), + width: width) + return item + } + + private func formatVersionLabel(_ version: String) -> String { + let trimmed = version.trimmingCharacters(in: .whitespacesAndNewlines) + guard !trimmed.isEmpty else { return version } + if trimmed.hasPrefix("v") { return trimmed } + if let first = trimmed.unicodeScalars.first, CharacterSet.decimalDigits.contains(first) { + return "v\(trimmed)" + } + return trimmed + } + + @objc + private func patchThinking(_ sender: NSMenuItem) { + guard let dict = sender.representedObject as? [String: Any], + let key = dict["key"] as? String + else { return } + let value = dict["value"] as? String + Task { + do { + try await SessionActions.patchSession(key: key, thinking: .some(value)) + await self.refreshCache(force: true) + } catch { + await MainActor.run { + SessionActions.presentError(title: "Update thinking failed", error: error) + } + } + } + } + + @objc + private func patchVerbose(_ sender: NSMenuItem) { + guard let dict = sender.representedObject as? [String: Any], + let key = dict["key"] as? String + else { return } + let value = dict["value"] as? String + Task { + do { + try await SessionActions.patchSession(key: key, verbose: .some(value)) + await self.refreshCache(force: true) + } catch { + await MainActor.run { + SessionActions.presentError(title: "Update verbose failed", error: error) + } + } + } + } + + @objc + private func openSessionLog(_ sender: NSMenuItem) { + guard let dict = sender.representedObject as? [String: String], + let sessionId = dict["sessionId"], + let storePath = dict["storePath"] + else { return } + SessionActions.openSessionLogInCode(sessionId: sessionId, storePath: storePath) + } + + @objc + private func resetSession(_ sender: NSMenuItem) { + guard let key = sender.representedObject as? String else { return } + Task { @MainActor in + guard SessionActions.confirmDestructiveAction( + title: "Reset session?", + message: "Starts a new session id for “\(key)”.", + action: "Reset") + else { return } + + do { + try await SessionActions.resetSession(key: key) + await self.refreshCache(force: true) + } catch { + SessionActions.presentError(title: "Reset failed", error: error) + } + } + } + + @objc + private func compactSession(_ sender: NSMenuItem) { + guard let key = sender.representedObject as? String else { return } + Task { @MainActor in + guard SessionActions.confirmDestructiveAction( + title: "Compact session log?", + message: "Keeps the last 400 lines; archives the old file.", + action: "Compact") + else { return } + + do { + try await SessionActions.compactSession(key: key, maxLines: 400) + await self.refreshCache(force: true) + } catch { + SessionActions.presentError(title: "Compact failed", error: error) + } + } + } + + @objc + private func deleteSession(_ sender: NSMenuItem) { + guard let key = sender.representedObject as? String else { return } + Task { @MainActor in + guard SessionActions.confirmDestructiveAction( + title: "Delete session?", + message: "Deletes the “\(key)” entry and archives its transcript.", + action: "Delete") + else { return } + + do { + try await SessionActions.deleteSession(key: key) + await self.refreshCache(force: true) + } catch { + SessionActions.presentError(title: "Delete failed", error: error) + } + } + } + + @objc + private func copyNodeSummary(_ sender: NSMenuItem) { + guard let summary = sender.representedObject as? String else { return } + NSPasteboard.general.clearContents() + NSPasteboard.general.setString(summary, forType: .string) + } + + @objc + private func copyNodeValue(_ sender: NSMenuItem) { + guard let value = sender.representedObject as? String else { return } + NSPasteboard.general.clearContents() + NSPasteboard.general.setString(value, forType: .string) + } +} + +extension MenuSessionsInjector { + // MARK: - Width + placement + + private func findInsertIndex(in menu: NSMenu) -> Int? { + // Insert right before the separator above "Send Heartbeats". + if let idx = menu.items.firstIndex(where: { $0.title == "Send Heartbeats" }) { + if let sepIdx = menu.items[..= 1 { return 1 } + return menu.items.count + } + + private func findNodesInsertIndex(in menu: NSMenu) -> Int? { + if let idx = menu.items.firstIndex(where: { $0.title == "Send Heartbeats" }) { + if let sepIdx = menu.items[..= 1 { return 1 } + return menu.items.count + } + + private func initialWidth(for menu: NSMenu) -> CGFloat { + if let openWidth = self.menuOpenWidth { + return max(300, openWidth) + } + return self.currentMenuWidth(for: menu) + } + + private func submenuWidth() -> CGFloat { + if let openWidth = self.menuOpenWidth { + return max(300, openWidth) + } + if let cached = self.lastKnownMenuWidth { + return max(300, cached) + } + return self.fallbackWidth + } + + private func menuWindowWidth(for menu: NSMenu) -> CGFloat? { + var menuWindow: NSWindow? + for item in menu.items { + if let window = item.view?.window { + menuWindow = window + break + } + } + guard let width = menuWindow?.contentView?.bounds.width, width > 0 else { return nil } + return width + } + + private func sortedNodeEntries() -> [NodeInfo] { + let entries = self.nodesStore.nodes.filter(\.isConnected) + return entries.sorted { lhs, rhs in + if lhs.isConnected != rhs.isConnected { return lhs.isConnected } + if lhs.isPaired != rhs.isPaired { return lhs.isPaired } + let lhsName = NodeMenuEntryFormatter.primaryName(lhs).lowercased() + let rhsName = NodeMenuEntryFormatter.primaryName(rhs).lowercased() + if lhsName == rhsName { return lhs.nodeId < rhs.nodeId } + return lhsName < rhsName + } + } +} + +extension MenuSessionsInjector { + // MARK: - Views + + private func makeHostedView(rootView: AnyView, width: CGFloat, highlighted: Bool) -> NSView { + if highlighted { + let container = HighlightedMenuItemHostView(rootView: rootView, width: width) + return container + } + + let hosting = NSHostingView(rootView: rootView) + hosting.frame.size.width = max(1, width) + let size = hosting.fittingSize + hosting.frame = NSRect(origin: .zero, size: NSSize(width: width, height: size.height)) + return hosting + } + + private func captureMenuWidthIfAvailable(from view: NSView) { + guard !self.isMenuOpen else { return } + guard let width = view.window?.contentView?.bounds.width, width > 0 else { return } + self.lastKnownMenuWidth = max(300, width) + } + + private func currentMenuWidth(for menu: NSMenu) -> CGFloat { + if let width = self.menuWindowWidth(for: menu) { + return max(300, width) + } + let candidates: [CGFloat] = [ + menu.size.width, + menu.minimumWidth, + self.lastKnownMenuWidth ?? 0, + self.fallbackWidth, + ] + let resolved = candidates.max() ?? self.fallbackWidth + return max(300, resolved) + } +} + +#if DEBUG +extension MenuSessionsInjector { + func setTestingControlChannelConnected(_ connected: Bool?) { + self.testControlChannelConnected = connected + } + + func setTestingSnapshot(_ snapshot: SessionStoreSnapshot?, errorText: String? = nil) { + self.cachedSnapshot = snapshot + self.cachedErrorText = errorText + self.cacheUpdatedAt = Date() + } + + func setTestingUsageSummary(_ summary: GatewayUsageSummary?, errorText: String? = nil) { + self.cachedUsageSummary = summary + self.cachedUsageErrorText = errorText + self.usageCacheUpdatedAt = Date() + } + + func injectForTesting(into menu: NSMenu) { + self.inject(into: menu) + } +} +#endif diff --git a/apps/macos/Sources/OpenClaw/MenuUsageHeaderView.swift b/apps/macos/Sources/OpenClaw/MenuUsageHeaderView.swift new file mode 100644 index 0000000000000000000000000000000000000000..dbb717d690a6882fd3f1b0b6162e112b73c2a942 --- /dev/null +++ b/apps/macos/Sources/OpenClaw/MenuUsageHeaderView.swift @@ -0,0 +1,35 @@ +import SwiftUI + +struct MenuUsageHeaderView: View { + let count: Int + + private let paddingTop: CGFloat = 8 + private let paddingBottom: CGFloat = 6 + private let paddingTrailing: CGFloat = 10 + private let paddingLeading: CGFloat = 20 + + var body: some View { + VStack(alignment: .leading, spacing: 6) { + HStack(alignment: .firstTextBaseline) { + Text("Usage") + .font(.caption.weight(.semibold)) + .foregroundStyle(.secondary) + Spacer(minLength: 10) + Text(self.subtitle) + .font(.caption) + .foregroundStyle(.secondary) + } + } + .padding(.top, self.paddingTop) + .padding(.bottom, self.paddingBottom) + .padding(.leading, self.paddingLeading) + .padding(.trailing, self.paddingTrailing) + .frame(minWidth: 300, maxWidth: .infinity, alignment: .leading) + .transaction { txn in txn.animation = nil } + } + + private var subtitle: String { + if self.count == 1 { return "1 provider" } + return "\(self.count) providers" + } +} diff --git a/apps/macos/Sources/OpenClaw/MicLevelMonitor.swift b/apps/macos/Sources/OpenClaw/MicLevelMonitor.swift new file mode 100644 index 0000000000000000000000000000000000000000..af72740a676f5723d18538f9e5ba975bcbe65e5f --- /dev/null +++ b/apps/macos/Sources/OpenClaw/MicLevelMonitor.swift @@ -0,0 +1,97 @@ +import AVFoundation +import OSLog +import SwiftUI + +actor MicLevelMonitor { + private let logger = Logger(subsystem: "ai.openclaw", category: "voicewake.meter") + private var engine: AVAudioEngine? + private var update: (@Sendable (Double) -> Void)? + private var running = false + private var smoothedLevel: Double = 0 + + func start(onLevel: @Sendable @escaping (Double) -> Void) async throws { + self.update = onLevel + if self.running { return } + self.logger.info( + "mic level monitor start (\(AudioInputDeviceObserver.defaultInputDeviceSummary(), privacy: .public))") + let engine = AVAudioEngine() + self.engine = engine + let input = engine.inputNode + let format = input.outputFormat(forBus: 0) + guard format.channelCount > 0, format.sampleRate > 0 else { + self.engine = nil + throw NSError( + domain: "MicLevelMonitor", + code: 1, + userInfo: [NSLocalizedDescriptionKey: "No audio input available"]) + } + input.removeTap(onBus: 0) + input.installTap(onBus: 0, bufferSize: 512, format: format) { [weak self] buffer, _ in + guard let self else { return } + let level = Self.normalizedLevel(from: buffer) + Task { await self.push(level: level) } + } + engine.prepare() + try engine.start() + self.running = true + } + + func stop() { + guard self.running else { return } + if let engine { + engine.inputNode.removeTap(onBus: 0) + engine.stop() + } + self.engine = nil + self.running = false + } + + private func push(level: Double) { + self.smoothedLevel = (self.smoothedLevel * 0.45) + (level * 0.55) + guard let update else { return } + let value = self.smoothedLevel + Task { @MainActor in update(value) } + } + + private static func normalizedLevel(from buffer: AVAudioPCMBuffer) -> Double { + guard let channel = buffer.floatChannelData?[0] else { return 0 } + let frameCount = Int(buffer.frameLength) + guard frameCount > 0 else { return 0 } + var sum: Float = 0 + for i in 0.. Double(idx) + RoundedRectangle(cornerRadius: 2) + .fill(fill ? self.segmentColor(for: idx) : Color.gray.opacity(0.35)) + .frame(width: 14, height: 10) + } + } + .padding(4) + .background( + RoundedRectangle(cornerRadius: 6) + .stroke(Color.gray.opacity(0.25), lineWidth: 1)) + } + + private func segmentColor(for idx: Int) -> Color { + let fraction = Double(idx + 1) / Double(self.segments) + if fraction < 0.65 { return .green } + if fraction < 0.85 { return .yellow } + return .red + } +} diff --git a/apps/macos/Sources/OpenClaw/ModelCatalogLoader.swift b/apps/macos/Sources/OpenClaw/ModelCatalogLoader.swift new file mode 100644 index 0000000000000000000000000000000000000000..ff966e1eabcea9ac253ae3a5fc075baf04a22e7a --- /dev/null +++ b/apps/macos/Sources/OpenClaw/ModelCatalogLoader.swift @@ -0,0 +1,156 @@ +import Foundation +import JavaScriptCore + +enum ModelCatalogLoader { + static var defaultPath: String { self.resolveDefaultPath() } + private static let logger = Logger(subsystem: "ai.openclaw", category: "models") + private nonisolated static let appSupportDir: URL = { + let base = FileManager().urls(for: .applicationSupportDirectory, in: .userDomainMask).first! + return base.appendingPathComponent("OpenClaw", isDirectory: true) + }() + + private static var cachePath: URL { + self.appSupportDir.appendingPathComponent("model-catalog/models.generated.js", isDirectory: false) + } + + static func load(from path: String) async throws -> [ModelChoice] { + let expanded = (path as NSString).expandingTildeInPath + guard let resolved = self.resolvePath(preferred: expanded) else { + self.logger.error("model catalog load failed: file not found") + throw NSError( + domain: "ModelCatalogLoader", + code: 1, + userInfo: [NSLocalizedDescriptionKey: "Model catalog file not found"]) + } + self.logger.debug("model catalog load start file=\(URL(fileURLWithPath: resolved.path).lastPathComponent)") + let source = try String(contentsOfFile: resolved.path, encoding: .utf8) + let sanitized = self.sanitize(source: source) + + let ctx = JSContext() + ctx?.exceptionHandler = { _, exception in + if let exception { + self.logger.warning("model catalog JS exception: \(exception)") + } + } + ctx?.evaluateScript(sanitized) + guard let rawModels = ctx?.objectForKeyedSubscript("MODELS")?.toDictionary() as? [String: Any] else { + self.logger.error("model catalog parse failed: MODELS missing") + throw NSError( + domain: "ModelCatalogLoader", + code: 1, + userInfo: [NSLocalizedDescriptionKey: "Failed to parse models.generated.ts"]) + } + + var choices: [ModelChoice] = [] + for (provider, value) in rawModels { + guard let models = value as? [String: Any] else { continue } + for (id, payload) in models { + guard let dict = payload as? [String: Any] else { continue } + let name = dict["name"] as? String ?? id + let ctxWindow = dict["contextWindow"] as? Int + choices.append(ModelChoice(id: id, name: name, provider: provider, contextWindow: ctxWindow)) + } + } + + let sorted = choices.sorted { lhs, rhs in + if lhs.provider == rhs.provider { + return lhs.name.localizedCaseInsensitiveCompare(rhs.name) == .orderedAscending + } + return lhs.provider.localizedCaseInsensitiveCompare(rhs.provider) == .orderedAscending + } + self.logger.debug("model catalog loaded providers=\(rawModels.count) models=\(sorted.count)") + if resolved.shouldCache { + self.cacheCatalog(sourcePath: resolved.path) + } + return sorted + } + + private static func resolveDefaultPath() -> String { + let cache = self.cachePath.path + if FileManager().isReadableFile(atPath: cache) { return cache } + if let bundlePath = self.bundleCatalogPath() { return bundlePath } + if let nodePath = self.nodeModulesCatalogPath() { return nodePath } + return cache + } + + private static func resolvePath(preferred: String) -> (path: String, shouldCache: Bool)? { + if FileManager().isReadableFile(atPath: preferred) { + return (preferred, preferred != self.cachePath.path) + } + + if let bundlePath = self.bundleCatalogPath(), bundlePath != preferred { + self.logger.warning("model catalog path missing; falling back to bundled catalog") + return (bundlePath, true) + } + + let cache = self.cachePath.path + if cache != preferred, FileManager().isReadableFile(atPath: cache) { + self.logger.warning("model catalog path missing; falling back to cached catalog") + return (cache, false) + } + + if let nodePath = self.nodeModulesCatalogPath(), nodePath != preferred { + self.logger.warning("model catalog path missing; falling back to node_modules catalog") + return (nodePath, true) + } + + return nil + } + + private static func bundleCatalogPath() -> String? { + guard let url = Bundle.main.url(forResource: "models.generated", withExtension: "js") else { + return nil + } + return url.path + } + + private static func nodeModulesCatalogPath() -> String? { + let roots = [ + URL(fileURLWithPath: CommandResolver.projectRootPath()), + URL(fileURLWithPath: FileManager().currentDirectoryPath), + ] + for root in roots { + let candidate = root + .appendingPathComponent("node_modules/@mariozechner/pi-ai/dist/models.generated.js") + if FileManager().isReadableFile(atPath: candidate.path) { + return candidate.path + } + } + return nil + } + + private static func cacheCatalog(sourcePath: String) { + let destination = self.cachePath + do { + try FileManager().createDirectory( + at: destination.deletingLastPathComponent(), + withIntermediateDirectories: true) + if FileManager().fileExists(atPath: destination.path) { + try FileManager().removeItem(at: destination) + } + try FileManager().copyItem(atPath: sourcePath, toPath: destination.path) + self.logger.debug("model catalog cached file=\(destination.lastPathComponent)") + } catch { + self.logger.warning("model catalog cache failed: \(error.localizedDescription)") + } + } + + private static func sanitize(source: String) -> String { + guard let exportRange = source.range(of: "export const MODELS"), + let firstBrace = source[exportRange.upperBound...].firstIndex(of: "{"), + let lastBrace = source.lastIndex(of: "}") + else { + return "var MODELS = {}" + } + var body = String(source[firstBrace...lastBrace]) + body = body.replacingOccurrences( + of: #"(?m)\bsatisfies\s+[^,}\n]+"#, + with: "", + options: .regularExpression) + body = body.replacingOccurrences( + of: #"(?m)\bas\s+[^;,\n]+"#, + with: "", + options: .regularExpression) + return "var MODELS = \(body);" + } +} diff --git a/apps/macos/Sources/OpenClaw/NSAttributedString+VoiceWake.swift b/apps/macos/Sources/OpenClaw/NSAttributedString+VoiceWake.swift new file mode 100644 index 0000000000000000000000000000000000000000..cb4be425834b6dc814b1657d55cad8be925db11f --- /dev/null +++ b/apps/macos/Sources/OpenClaw/NSAttributedString+VoiceWake.swift @@ -0,0 +1,9 @@ +import Foundation + +extension NSAttributedString { + func strippingForegroundColor() -> NSAttributedString { + let mutable = NSMutableAttributedString(attributedString: self) + mutable.removeAttribute(.foregroundColor, range: NSRange(location: 0, length: mutable.length)) + return mutable + } +} diff --git a/apps/macos/Sources/OpenClaw/NodeMode/MacNodeLocationService.swift b/apps/macos/Sources/OpenClaw/NodeMode/MacNodeLocationService.swift new file mode 100644 index 0000000000000000000000000000000000000000..db404aa6e171f239cd83d4fec4de4d07a55938f4 --- /dev/null +++ b/apps/macos/Sources/OpenClaw/NodeMode/MacNodeLocationService.swift @@ -0,0 +1,139 @@ +import OpenClawKit +import CoreLocation +import Foundation + +@MainActor +final class MacNodeLocationService: NSObject, CLLocationManagerDelegate { + enum Error: Swift.Error { + case timeout + case unavailable + } + + private let manager = CLLocationManager() + private var locationContinuation: CheckedContinuation? + + override init() { + super.init() + self.manager.delegate = self + self.manager.desiredAccuracy = kCLLocationAccuracyBest + } + + func authorizationStatus() -> CLAuthorizationStatus { + self.manager.authorizationStatus + } + + func accuracyAuthorization() -> CLAccuracyAuthorization { + if #available(macOS 11.0, *) { + return self.manager.accuracyAuthorization + } + return .fullAccuracy + } + + func currentLocation( + desiredAccuracy: OpenClawLocationAccuracy, + maxAgeMs: Int?, + timeoutMs: Int?) async throws -> CLLocation + { + guard CLLocationManager.locationServicesEnabled() else { + throw Error.unavailable + } + + let now = Date() + if let maxAgeMs, + let cached = self.manager.location, + now.timeIntervalSince(cached.timestamp) * 1000 <= Double(maxAgeMs) + { + return cached + } + + self.manager.desiredAccuracy = Self.accuracyValue(desiredAccuracy) + let timeout = max(0, timeoutMs ?? 10000) + return try await self.withTimeout(timeoutMs: timeout) { + try await self.requestLocation() + } + } + + private func requestLocation() async throws -> CLLocation { + try await withCheckedThrowingContinuation { cont in + self.locationContinuation = cont + self.manager.requestLocation() + } + } + + private func withTimeout( + timeoutMs: Int, + operation: @escaping () async throws -> T) async throws -> T + { + if timeoutMs == 0 { + return try await operation() + } + + return try await withCheckedThrowingContinuation { continuation in + var didFinish = false + + func finish(returning value: T) { + guard !didFinish else { return } + didFinish = true + continuation.resume(returning: value) + } + + func finish(throwing error: Swift.Error) { + guard !didFinish else { return } + didFinish = true + continuation.resume(throwing: error) + } + + let timeoutItem = DispatchWorkItem { + finish(throwing: Error.timeout) + } + DispatchQueue.main.asyncAfter( + deadline: .now() + .milliseconds(timeoutMs), + execute: timeoutItem) + + Task { @MainActor in + do { + let value = try await operation() + timeoutItem.cancel() + finish(returning: value) + } catch { + timeoutItem.cancel() + finish(throwing: error) + } + } + } + } + + private static func accuracyValue(_ accuracy: OpenClawLocationAccuracy) -> CLLocationAccuracy { + switch accuracy { + case .coarse: + kCLLocationAccuracyKilometer + case .balanced: + kCLLocationAccuracyHundredMeters + case .precise: + kCLLocationAccuracyBest + } + } + + // MARK: - CLLocationManagerDelegate (nonisolated for Swift 6 compatibility) + + nonisolated func locationManager(_ manager: CLLocationManager, didUpdateLocations locations: [CLLocation]) { + Task { @MainActor in + guard let cont = self.locationContinuation else { return } + self.locationContinuation = nil + if let latest = locations.last { + cont.resume(returning: latest) + } else { + cont.resume(throwing: Error.unavailable) + } + } + } + + nonisolated func locationManager(_ manager: CLLocationManager, didFailWithError error: Swift.Error) { + let errorCopy = error // Capture error for Sendable compliance + Task { @MainActor in + guard let cont = self.locationContinuation else { return } + self.locationContinuation = nil + cont.resume(throwing: errorCopy) + } + } +} diff --git a/apps/macos/Sources/OpenClaw/NodeMode/MacNodeModeCoordinator.swift b/apps/macos/Sources/OpenClaw/NodeMode/MacNodeModeCoordinator.swift new file mode 100644 index 0000000000000000000000000000000000000000..eed0755f9b75c0cef5394c3f7a582da4a7bf8a1d --- /dev/null +++ b/apps/macos/Sources/OpenClaw/NodeMode/MacNodeModeCoordinator.swift @@ -0,0 +1,171 @@ +import OpenClawKit +import Foundation +import OSLog + +@MainActor +final class MacNodeModeCoordinator { + static let shared = MacNodeModeCoordinator() + + private let logger = Logger(subsystem: "ai.openclaw", category: "mac-node") + private var task: Task? + private let runtime = MacNodeRuntime() + private let session = GatewayNodeSession() + + func start() { + guard self.task == nil else { return } + self.task = Task { [weak self] in + await self?.run() + } + } + + func stop() { + self.task?.cancel() + self.task = nil + Task { await self.session.disconnect() } + } + + func setPreferredGatewayStableID(_ stableID: String?) { + GatewayDiscoveryPreferences.setPreferredStableID(stableID) + Task { await self.session.disconnect() } + } + + private func run() async { + var retryDelay: UInt64 = 1_000_000_000 + var lastCameraEnabled: Bool? + let defaults = UserDefaults.standard + + while !Task.isCancelled { + if await MainActor.run(body: { AppStateStore.shared.isPaused }) { + try? await Task.sleep(nanoseconds: 1_000_000_000) + continue + } + + let cameraEnabled = defaults.object(forKey: cameraEnabledKey) as? Bool ?? false + if lastCameraEnabled == nil { + lastCameraEnabled = cameraEnabled + } else if lastCameraEnabled != cameraEnabled { + lastCameraEnabled = cameraEnabled + await self.session.disconnect() + try? await Task.sleep(nanoseconds: 200_000_000) + } + + do { + let config = try await GatewayEndpointStore.shared.requireConfig() + let caps = self.currentCaps() + let commands = self.currentCommands(caps: caps) + let permissions = await self.currentPermissions() + let connectOptions = GatewayConnectOptions( + role: "node", + scopes: [], + caps: caps, + commands: commands, + permissions: permissions, + clientId: "openclaw-macos", + clientMode: "node", + clientDisplayName: InstanceIdentity.displayName) + let sessionBox = self.buildSessionBox(url: config.url) + + try await self.session.connect( + url: config.url, + token: config.token, + password: config.password, + connectOptions: connectOptions, + sessionBox: sessionBox, + onConnected: { [weak self] in + guard let self else { return } + self.logger.info("mac node connected to gateway") + let mainSessionKey = await GatewayConnection.shared.mainSessionKey() + await self.runtime.updateMainSessionKey(mainSessionKey) + await self.runtime.setEventSender { [weak self] event, payload in + guard let self else { return } + await self.session.sendEvent(event: event, payloadJSON: payload) + } + }, + onDisconnected: { [weak self] reason in + guard let self else { return } + await self.runtime.setEventSender(nil) + self.logger.error("mac node disconnected: \(reason, privacy: .public)") + }, + onInvoke: { [weak self] req in + guard let self else { + return BridgeInvokeResponse( + id: req.id, + ok: false, + error: OpenClawNodeError(code: .unavailable, message: "UNAVAILABLE: node not ready")) + } + return await self.runtime.handleInvoke(req) + }) + + retryDelay = 1_000_000_000 + try? await Task.sleep(nanoseconds: 1_000_000_000) + } catch { + self.logger.error("mac node gateway connect failed: \(error.localizedDescription, privacy: .public)") + try? await Task.sleep(nanoseconds: min(retryDelay, 10_000_000_000)) + retryDelay = min(retryDelay * 2, 10_000_000_000) + } + } + } + + private func currentCaps() -> [String] { + var caps: [String] = [OpenClawCapability.canvas.rawValue, OpenClawCapability.screen.rawValue] + if UserDefaults.standard.object(forKey: cameraEnabledKey) as? Bool ?? false { + caps.append(OpenClawCapability.camera.rawValue) + } + let rawLocationMode = UserDefaults.standard.string(forKey: locationModeKey) ?? "off" + if OpenClawLocationMode(rawValue: rawLocationMode) != .off { + caps.append(OpenClawCapability.location.rawValue) + } + return caps + } + + private func currentPermissions() async -> [String: Bool] { + let statuses = await PermissionManager.status() + return Dictionary(uniqueKeysWithValues: statuses.map { ($0.key.rawValue, $0.value) }) + } + + private func currentCommands(caps: [String]) -> [String] { + var commands: [String] = [ + OpenClawCanvasCommand.present.rawValue, + OpenClawCanvasCommand.hide.rawValue, + OpenClawCanvasCommand.navigate.rawValue, + OpenClawCanvasCommand.evalJS.rawValue, + OpenClawCanvasCommand.snapshot.rawValue, + OpenClawCanvasA2UICommand.push.rawValue, + OpenClawCanvasA2UICommand.pushJSONL.rawValue, + OpenClawCanvasA2UICommand.reset.rawValue, + MacNodeScreenCommand.record.rawValue, + OpenClawSystemCommand.notify.rawValue, + OpenClawSystemCommand.which.rawValue, + OpenClawSystemCommand.run.rawValue, + OpenClawSystemCommand.execApprovalsGet.rawValue, + OpenClawSystemCommand.execApprovalsSet.rawValue, + ] + + let capsSet = Set(caps) + if capsSet.contains(OpenClawCapability.camera.rawValue) { + commands.append(OpenClawCameraCommand.list.rawValue) + commands.append(OpenClawCameraCommand.snap.rawValue) + commands.append(OpenClawCameraCommand.clip.rawValue) + } + if capsSet.contains(OpenClawCapability.location.rawValue) { + commands.append(OpenClawLocationCommand.get.rawValue) + } + + return commands + } + + private func buildSessionBox(url: URL) -> WebSocketSessionBox? { + guard url.scheme?.lowercased() == "wss" else { return nil } + let host = url.host ?? "gateway" + let port = url.port ?? 443 + let stableID = "\(host):\(port)" + let stored = GatewayTLSStore.loadFingerprint(stableID: stableID) + let params = GatewayTLSParams( + required: true, + expectedFingerprint: stored, + allowTOFU: stored == nil, + storeKey: stableID) + let session = GatewayTLSPinningSession(params: params) + return WebSocketSessionBox(session: session) + } +} diff --git a/apps/macos/Sources/OpenClaw/NodeMode/MacNodeRuntime.swift b/apps/macos/Sources/OpenClaw/NodeMode/MacNodeRuntime.swift new file mode 100644 index 0000000000000000000000000000000000000000..0b88f159098edddca4c83b3fa7afefde78157ea5 --- /dev/null +++ b/apps/macos/Sources/OpenClaw/NodeMode/MacNodeRuntime.swift @@ -0,0 +1,969 @@ +import AppKit +import OpenClawIPC +import OpenClawKit +import Foundation + +actor MacNodeRuntime { + private let cameraCapture = CameraCaptureService() + private let makeMainActorServices: () async -> any MacNodeRuntimeMainActorServices + private var cachedMainActorServices: (any MacNodeRuntimeMainActorServices)? + private var mainSessionKey: String = "main" + private var eventSender: (@Sendable (String, String?) async -> Void)? + + init( + makeMainActorServices: @escaping () async -> any MacNodeRuntimeMainActorServices = { + await MainActor.run { LiveMacNodeRuntimeMainActorServices() } + }) + { + self.makeMainActorServices = makeMainActorServices + } + + func updateMainSessionKey(_ sessionKey: String) { + let trimmed = sessionKey.trimmingCharacters(in: .whitespacesAndNewlines) + guard !trimmed.isEmpty else { return } + self.mainSessionKey = trimmed + } + + func setEventSender(_ sender: (@Sendable (String, String?) async -> Void)?) { + self.eventSender = sender + } + + func handleInvoke(_ req: BridgeInvokeRequest) async -> BridgeInvokeResponse { + let command = req.command + if self.isCanvasCommand(command), !Self.canvasEnabled() { + return BridgeInvokeResponse( + id: req.id, + ok: false, + error: OpenClawNodeError( + code: .unavailable, + message: "CANVAS_DISABLED: enable Canvas in Settings")) + } + do { + switch command { + case OpenClawCanvasCommand.present.rawValue, + OpenClawCanvasCommand.hide.rawValue, + OpenClawCanvasCommand.navigate.rawValue, + OpenClawCanvasCommand.evalJS.rawValue, + OpenClawCanvasCommand.snapshot.rawValue: + return try await self.handleCanvasInvoke(req) + case OpenClawCanvasA2UICommand.reset.rawValue, + OpenClawCanvasA2UICommand.push.rawValue, + OpenClawCanvasA2UICommand.pushJSONL.rawValue: + return try await self.handleA2UIInvoke(req) + case OpenClawCameraCommand.snap.rawValue, + OpenClawCameraCommand.clip.rawValue, + OpenClawCameraCommand.list.rawValue: + return try await self.handleCameraInvoke(req) + case OpenClawLocationCommand.get.rawValue: + return try await self.handleLocationInvoke(req) + case MacNodeScreenCommand.record.rawValue: + return try await self.handleScreenRecordInvoke(req) + case OpenClawSystemCommand.run.rawValue: + return try await self.handleSystemRun(req) + case OpenClawSystemCommand.which.rawValue: + return try await self.handleSystemWhich(req) + case OpenClawSystemCommand.notify.rawValue: + return try await self.handleSystemNotify(req) + case OpenClawSystemCommand.execApprovalsGet.rawValue: + return try await self.handleSystemExecApprovalsGet(req) + case OpenClawSystemCommand.execApprovalsSet.rawValue: + return try await self.handleSystemExecApprovalsSet(req) + default: + return Self.errorResponse(req, code: .invalidRequest, message: "INVALID_REQUEST: unknown command") + } + } catch { + return Self.errorResponse(req, code: .unavailable, message: error.localizedDescription) + } + } + + private func isCanvasCommand(_ command: String) -> Bool { + command.hasPrefix("canvas.") || command.hasPrefix("canvas.a2ui.") + } + + private func handleCanvasInvoke(_ req: BridgeInvokeRequest) async throws -> BridgeInvokeResponse { + switch req.command { + case OpenClawCanvasCommand.present.rawValue: + let params = (try? Self.decodeParams(OpenClawCanvasPresentParams.self, from: req.paramsJSON)) ?? + OpenClawCanvasPresentParams() + let urlTrimmed = params.url?.trimmingCharacters(in: .whitespacesAndNewlines) ?? "" + let url = urlTrimmed.isEmpty ? nil : urlTrimmed + let placement = params.placement.map { + CanvasPlacement(x: $0.x, y: $0.y, width: $0.width, height: $0.height) + } + let sessionKey = self.mainSessionKey + try await MainActor.run { + _ = try CanvasManager.shared.showDetailed( + sessionKey: sessionKey, + target: url, + placement: placement) + } + return BridgeInvokeResponse(id: req.id, ok: true) + case OpenClawCanvasCommand.hide.rawValue: + let sessionKey = self.mainSessionKey + await MainActor.run { + CanvasManager.shared.hide(sessionKey: sessionKey) + } + return BridgeInvokeResponse(id: req.id, ok: true) + case OpenClawCanvasCommand.navigate.rawValue: + let params = try Self.decodeParams(OpenClawCanvasNavigateParams.self, from: req.paramsJSON) + let sessionKey = self.mainSessionKey + try await MainActor.run { + _ = try CanvasManager.shared.show(sessionKey: sessionKey, path: params.url) + } + return BridgeInvokeResponse(id: req.id, ok: true) + case OpenClawCanvasCommand.evalJS.rawValue: + let params = try Self.decodeParams(OpenClawCanvasEvalParams.self, from: req.paramsJSON) + let sessionKey = self.mainSessionKey + let result = try await CanvasManager.shared.eval( + sessionKey: sessionKey, + javaScript: params.javaScript) + let payload = try Self.encodePayload(["result": result] as [String: String]) + return BridgeInvokeResponse(id: req.id, ok: true, payloadJSON: payload) + case OpenClawCanvasCommand.snapshot.rawValue: + let params = try? Self.decodeParams(OpenClawCanvasSnapshotParams.self, from: req.paramsJSON) + let format = params?.format ?? .jpeg + let maxWidth: Int? = { + if let raw = params?.maxWidth, raw > 0 { return raw } + return switch format { + case .png: 900 + case .jpeg: 1600 + } + }() + let quality = params?.quality ?? 0.9 + + let sessionKey = self.mainSessionKey + let path = try await CanvasManager.shared.snapshot(sessionKey: sessionKey, outPath: nil) + defer { try? FileManager().removeItem(atPath: path) } + let data = try Data(contentsOf: URL(fileURLWithPath: path)) + guard let image = NSImage(data: data) else { + return Self.errorResponse(req, code: .unavailable, message: "canvas snapshot decode failed") + } + let encoded = try Self.encodeCanvasSnapshot( + image: image, + format: format, + maxWidth: maxWidth, + quality: quality) + let payload = try Self.encodePayload([ + "format": format == .jpeg ? "jpeg" : "png", + "base64": encoded.base64EncodedString(), + ]) + return BridgeInvokeResponse(id: req.id, ok: true, payloadJSON: payload) + default: + return Self.errorResponse(req, code: .invalidRequest, message: "INVALID_REQUEST: unknown command") + } + } + + private func handleA2UIInvoke(_ req: BridgeInvokeRequest) async throws -> BridgeInvokeResponse { + switch req.command { + case OpenClawCanvasA2UICommand.reset.rawValue: + try await self.handleA2UIReset(req) + case OpenClawCanvasA2UICommand.push.rawValue, + OpenClawCanvasA2UICommand.pushJSONL.rawValue: + try await self.handleA2UIPush(req) + default: + Self.errorResponse(req, code: .invalidRequest, message: "INVALID_REQUEST: unknown command") + } + } + + private func handleCameraInvoke(_ req: BridgeInvokeRequest) async throws -> BridgeInvokeResponse { + guard Self.cameraEnabled() else { + return BridgeInvokeResponse( + id: req.id, + ok: false, + error: OpenClawNodeError( + code: .unavailable, + message: "CAMERA_DISABLED: enable Camera in Settings")) + } + switch req.command { + case OpenClawCameraCommand.snap.rawValue: + let params = (try? Self.decodeParams(OpenClawCameraSnapParams.self, from: req.paramsJSON)) ?? + OpenClawCameraSnapParams() + let delayMs = min(10000, max(0, params.delayMs ?? 2000)) + let res = try await self.cameraCapture.snap( + facing: CameraFacing(rawValue: params.facing?.rawValue ?? "") ?? .front, + maxWidth: params.maxWidth, + quality: params.quality, + deviceId: params.deviceId, + delayMs: delayMs) + struct SnapPayload: Encodable { + var format: String + var base64: String + var width: Int + var height: Int + } + let payload = try Self.encodePayload(SnapPayload( + format: (params.format ?? .jpg).rawValue, + base64: res.data.base64EncodedString(), + width: Int(res.size.width), + height: Int(res.size.height))) + return BridgeInvokeResponse(id: req.id, ok: true, payloadJSON: payload) + case OpenClawCameraCommand.clip.rawValue: + let params = (try? Self.decodeParams(OpenClawCameraClipParams.self, from: req.paramsJSON)) ?? + OpenClawCameraClipParams() + let res = try await self.cameraCapture.clip( + facing: CameraFacing(rawValue: params.facing?.rawValue ?? "") ?? .front, + durationMs: params.durationMs, + includeAudio: params.includeAudio ?? true, + deviceId: params.deviceId, + outPath: nil) + defer { try? FileManager().removeItem(atPath: res.path) } + let data = try Data(contentsOf: URL(fileURLWithPath: res.path)) + struct ClipPayload: Encodable { + var format: String + var base64: String + var durationMs: Int + var hasAudio: Bool + } + let payload = try Self.encodePayload(ClipPayload( + format: (params.format ?? .mp4).rawValue, + base64: data.base64EncodedString(), + durationMs: res.durationMs, + hasAudio: res.hasAudio)) + return BridgeInvokeResponse(id: req.id, ok: true, payloadJSON: payload) + case OpenClawCameraCommand.list.rawValue: + let devices = await self.cameraCapture.listDevices() + let payload = try Self.encodePayload(["devices": devices]) + return BridgeInvokeResponse(id: req.id, ok: true, payloadJSON: payload) + default: + return Self.errorResponse(req, code: .invalidRequest, message: "INVALID_REQUEST: unknown command") + } + } + + private func handleLocationInvoke(_ req: BridgeInvokeRequest) async throws -> BridgeInvokeResponse { + let mode = Self.locationMode() + guard mode != .off else { + return BridgeInvokeResponse( + id: req.id, + ok: false, + error: OpenClawNodeError( + code: .unavailable, + message: "LOCATION_DISABLED: enable Location in Settings")) + } + let params = (try? Self.decodeParams(OpenClawLocationGetParams.self, from: req.paramsJSON)) ?? + OpenClawLocationGetParams() + let desired = params.desiredAccuracy ?? + (Self.locationPreciseEnabled() ? .precise : .balanced) + let services = await self.mainActorServices() + let status = await services.locationAuthorizationStatus() + let hasPermission = switch mode { + case .always: + status == .authorizedAlways + case .whileUsing: + status == .authorizedAlways + case .off: + false + } + if !hasPermission { + return BridgeInvokeResponse( + id: req.id, + ok: false, + error: OpenClawNodeError( + code: .unavailable, + message: "LOCATION_PERMISSION_REQUIRED: grant Location permission")) + } + do { + let location = try await services.currentLocation( + desiredAccuracy: desired, + maxAgeMs: params.maxAgeMs, + timeoutMs: params.timeoutMs) + let isPrecise = await services.locationAccuracyAuthorization() == .fullAccuracy + let payload = OpenClawLocationPayload( + lat: location.coordinate.latitude, + lon: location.coordinate.longitude, + accuracyMeters: location.horizontalAccuracy, + altitudeMeters: location.verticalAccuracy >= 0 ? location.altitude : nil, + speedMps: location.speed >= 0 ? location.speed : nil, + headingDeg: location.course >= 0 ? location.course : nil, + timestamp: ISO8601DateFormatter().string(from: location.timestamp), + isPrecise: isPrecise, + source: nil) + let json = try Self.encodePayload(payload) + return BridgeInvokeResponse(id: req.id, ok: true, payloadJSON: json) + } catch MacNodeLocationService.Error.timeout { + return BridgeInvokeResponse( + id: req.id, + ok: false, + error: OpenClawNodeError( + code: .unavailable, + message: "LOCATION_TIMEOUT: no fix in time")) + } catch { + return BridgeInvokeResponse( + id: req.id, + ok: false, + error: OpenClawNodeError( + code: .unavailable, + message: "LOCATION_UNAVAILABLE: \(error.localizedDescription)")) + } + } + + private func handleScreenRecordInvoke(_ req: BridgeInvokeRequest) async throws -> BridgeInvokeResponse { + let params = (try? Self.decodeParams(MacNodeScreenRecordParams.self, from: req.paramsJSON)) ?? + MacNodeScreenRecordParams() + if let format = params.format?.lowercased(), !format.isEmpty, format != "mp4" { + return Self.errorResponse( + req, + code: .invalidRequest, + message: "INVALID_REQUEST: screen format must be mp4") + } + let services = await self.mainActorServices() + let res = try await services.recordScreen( + screenIndex: params.screenIndex, + durationMs: params.durationMs, + fps: params.fps, + includeAudio: params.includeAudio, + outPath: nil) + defer { try? FileManager().removeItem(atPath: res.path) } + let data = try Data(contentsOf: URL(fileURLWithPath: res.path)) + struct ScreenPayload: Encodable { + var format: String + var base64: String + var durationMs: Int? + var fps: Double? + var screenIndex: Int? + var hasAudio: Bool + } + let payload = try Self.encodePayload(ScreenPayload( + format: "mp4", + base64: data.base64EncodedString(), + durationMs: params.durationMs, + fps: params.fps, + screenIndex: params.screenIndex, + hasAudio: res.hasAudio)) + return BridgeInvokeResponse(id: req.id, ok: true, payloadJSON: payload) + } + + private func mainActorServices() async -> any MacNodeRuntimeMainActorServices { + if let cachedMainActorServices { return cachedMainActorServices } + let services = await self.makeMainActorServices() + self.cachedMainActorServices = services + return services + } + + private func handleA2UIReset(_ req: BridgeInvokeRequest) async throws -> BridgeInvokeResponse { + try await self.ensureA2UIHost() + + let sessionKey = self.mainSessionKey + let json = try await CanvasManager.shared.eval(sessionKey: sessionKey, javaScript: """ + (() => { + const host = globalThis.openclawA2UI; + if (!host) return JSON.stringify({ ok: false, error: "missing openclawA2UI" }); + return JSON.stringify(host.reset()); + })() + """) + return BridgeInvokeResponse(id: req.id, ok: true, payloadJSON: json) + } + + private func handleA2UIPush(_ req: BridgeInvokeRequest) async throws -> BridgeInvokeResponse { + let command = req.command + let messages: [OpenClawKit.AnyCodable] + if command == OpenClawCanvasA2UICommand.pushJSONL.rawValue { + let params = try Self.decodeParams(OpenClawCanvasA2UIPushJSONLParams.self, from: req.paramsJSON) + messages = try OpenClawCanvasA2UIJSONL.decodeMessagesFromJSONL(params.jsonl) + } else { + do { + let params = try Self.decodeParams(OpenClawCanvasA2UIPushParams.self, from: req.paramsJSON) + messages = params.messages + } catch { + let params = try Self.decodeParams(OpenClawCanvasA2UIPushJSONLParams.self, from: req.paramsJSON) + messages = try OpenClawCanvasA2UIJSONL.decodeMessagesFromJSONL(params.jsonl) + } + } + + try await self.ensureA2UIHost() + + let messagesJSON = try OpenClawCanvasA2UIJSONL.encodeMessagesJSONArray(messages) + let js = """ + (() => { + try { + const host = globalThis.openclawA2UI; + if (!host) return JSON.stringify({ ok: false, error: "missing openclawA2UI" }); + const messages = \(messagesJSON); + return JSON.stringify(host.applyMessages(messages)); + } catch (e) { + return JSON.stringify({ ok: false, error: String(e?.message ?? e) }); + } + })() + """ + let sessionKey = self.mainSessionKey + let resultJSON = try await CanvasManager.shared.eval(sessionKey: sessionKey, javaScript: js) + return BridgeInvokeResponse(id: req.id, ok: true, payloadJSON: resultJSON) + } + + private func ensureA2UIHost() async throws { + if await self.isA2UIReady() { return } + guard let a2uiUrl = await self.resolveA2UIHostUrl() else { + throw NSError(domain: "Canvas", code: 30, userInfo: [ + NSLocalizedDescriptionKey: "A2UI_HOST_NOT_CONFIGURED: gateway did not advertise canvas host", + ]) + } + let sessionKey = self.mainSessionKey + _ = try await MainActor.run { + try CanvasManager.shared.show(sessionKey: sessionKey, path: a2uiUrl) + } + if await self.isA2UIReady(poll: true) { return } + throw NSError(domain: "Canvas", code: 31, userInfo: [ + NSLocalizedDescriptionKey: "A2UI_HOST_UNAVAILABLE: A2UI host not reachable", + ]) + } + + private func resolveA2UIHostUrl() async -> String? { + guard let raw = await GatewayConnection.shared.canvasHostUrl() else { return nil } + let trimmed = raw.trimmingCharacters(in: .whitespacesAndNewlines) + guard !trimmed.isEmpty, let baseUrl = URL(string: trimmed) else { return nil } + return baseUrl.appendingPathComponent("__openclaw__/a2ui/").absoluteString + "?platform=macos" + } + + private func isA2UIReady(poll: Bool = false) async -> Bool { + let deadline = poll ? Date().addingTimeInterval(6.0) : Date() + while true { + do { + let sessionKey = self.mainSessionKey + let ready = try await CanvasManager.shared.eval(sessionKey: sessionKey, javaScript: """ + (() => { + const host = globalThis.openclawA2UI; + return String(Boolean(host)); + })() + """) + let trimmed = ready.trimmingCharacters(in: .whitespacesAndNewlines) + if trimmed == "true" { return true } + } catch { + // Ignore transient eval failures while the page is loading. + } + + guard poll, Date() < deadline else { return false } + try? await Task.sleep(nanoseconds: 120_000_000) + } + } + + private func handleSystemRun(_ req: BridgeInvokeRequest) async throws -> BridgeInvokeResponse { + let params = try Self.decodeParams(OpenClawSystemRunParams.self, from: req.paramsJSON) + let command = params.command + guard !command.isEmpty else { + return Self.errorResponse(req, code: .invalidRequest, message: "INVALID_REQUEST: command required") + } + let displayCommand = ExecCommandFormatter.displayString(for: command, rawCommand: params.rawCommand) + + let trimmedAgent = params.agentId?.trimmingCharacters(in: .whitespacesAndNewlines) ?? "" + let agentId = trimmedAgent.isEmpty ? nil : trimmedAgent + let approvals = ExecApprovalsStore.resolve(agentId: agentId) + let security = approvals.agent.security + let ask = approvals.agent.ask + let autoAllowSkills = approvals.agent.autoAllowSkills + let sessionKey = (params.sessionKey?.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty == false) + ? params.sessionKey!.trimmingCharacters(in: .whitespacesAndNewlines) + : self.mainSessionKey + let runId = UUID().uuidString + let env = Self.sanitizedEnv(params.env) + let resolution = ExecCommandResolution.resolve( + command: command, + rawCommand: params.rawCommand, + cwd: params.cwd, + env: env) + let allowlistMatch = security == .allowlist + ? ExecAllowlistMatcher.match(entries: approvals.allowlist, resolution: resolution) + : nil + let skillAllow: Bool + if autoAllowSkills, let name = resolution?.executableName { + let bins = await SkillBinsCache.shared.currentBins() + skillAllow = bins.contains(name) + } else { + skillAllow = false + } + + if security == .deny { + await self.emitExecEvent( + "exec.denied", + payload: ExecEventPayload( + sessionKey: sessionKey, + runId: runId, + host: "node", + command: displayCommand, + reason: "security=deny")) + return Self.errorResponse( + req, + code: .unavailable, + message: "SYSTEM_RUN_DISABLED: security=deny") + } + + let approval = await self.resolveSystemRunApproval( + req: req, + params: params, + context: ExecRunContext( + displayCommand: displayCommand, + security: security, + ask: ask, + agentId: agentId, + resolution: resolution, + allowlistMatch: allowlistMatch, + skillAllow: skillAllow, + sessionKey: sessionKey, + runId: runId)) + if let response = approval.response { return response } + let approvedByAsk = approval.approvedByAsk + let persistAllowlist = approval.persistAllowlist + if persistAllowlist, security == .allowlist, + let pattern = ExecApprovalHelpers.allowlistPattern(command: command, resolution: resolution) + { + ExecApprovalsStore.addAllowlistEntry(agentId: agentId, pattern: pattern) + } + + if security == .allowlist, allowlistMatch == nil, !skillAllow, !approvedByAsk { + await self.emitExecEvent( + "exec.denied", + payload: ExecEventPayload( + sessionKey: sessionKey, + runId: runId, + host: "node", + command: displayCommand, + reason: "allowlist-miss")) + return Self.errorResponse( + req, + code: .unavailable, + message: "SYSTEM_RUN_DENIED: allowlist miss") + } + + if let match = allowlistMatch { + ExecApprovalsStore.recordAllowlistUse( + agentId: agentId, + pattern: match.pattern, + command: displayCommand, + resolvedPath: resolution?.resolvedPath) + } + + if params.needsScreenRecording == true { + let authorized = await PermissionManager + .status([.screenRecording])[.screenRecording] ?? false + if !authorized { + await self.emitExecEvent( + "exec.denied", + payload: ExecEventPayload( + sessionKey: sessionKey, + runId: runId, + host: "node", + command: displayCommand, + reason: "permission:screenRecording")) + return Self.errorResponse( + req, + code: .unavailable, + message: "PERMISSION_MISSING: screenRecording") + } + } + + let timeoutSec = params.timeoutMs.flatMap { Double($0) / 1000.0 } + await self.emitExecEvent( + "exec.started", + payload: ExecEventPayload( + sessionKey: sessionKey, + runId: runId, + host: "node", + command: displayCommand)) + let result = await ShellExecutor.runDetailed( + command: command, + cwd: params.cwd, + env: env, + timeout: timeoutSec) + let combined = [result.stdout, result.stderr, result.errorMessage] + .compactMap(\.self) + .filter { !$0.isEmpty } + .joined(separator: "\n") + await self.emitExecEvent( + "exec.finished", + payload: ExecEventPayload( + sessionKey: sessionKey, + runId: runId, + host: "node", + command: displayCommand, + exitCode: result.exitCode, + timedOut: result.timedOut, + success: result.success, + output: ExecEventPayload.truncateOutput(combined))) + + struct RunPayload: Encodable { + var exitCode: Int? + var timedOut: Bool + var success: Bool + var stdout: String + var stderr: String + var error: String? + } + + let payload = try Self.encodePayload(RunPayload( + exitCode: result.exitCode, + timedOut: result.timedOut, + success: result.success, + stdout: result.stdout, + stderr: result.stderr, + error: result.errorMessage)) + return BridgeInvokeResponse(id: req.id, ok: true, payloadJSON: payload) + } + + private func handleSystemWhich(_ req: BridgeInvokeRequest) async throws -> BridgeInvokeResponse { + let params = try Self.decodeParams(OpenClawSystemWhichParams.self, from: req.paramsJSON) + let bins = params.bins + .map { $0.trimmingCharacters(in: .whitespacesAndNewlines) } + .filter { !$0.isEmpty } + guard !bins.isEmpty else { + return Self.errorResponse(req, code: .invalidRequest, message: "INVALID_REQUEST: bins required") + } + + let searchPaths = CommandResolver.preferredPaths() + var matches: [String] = [] + var paths: [String: String] = [:] + for bin in bins { + if let path = CommandResolver.findExecutable(named: bin, searchPaths: searchPaths) { + matches.append(bin) + paths[bin] = path + } + } + + struct WhichPayload: Encodable { + let bins: [String] + let paths: [String: String] + } + let payload = try Self.encodePayload(WhichPayload(bins: matches, paths: paths)) + return BridgeInvokeResponse(id: req.id, ok: true, payloadJSON: payload) + } + + private struct ExecApprovalOutcome { + var approvedByAsk: Bool + var persistAllowlist: Bool + var response: BridgeInvokeResponse? + } + + private struct ExecRunContext { + var displayCommand: String + var security: ExecSecurity + var ask: ExecAsk + var agentId: String? + var resolution: ExecCommandResolution? + var allowlistMatch: ExecAllowlistEntry? + var skillAllow: Bool + var sessionKey: String + var runId: String + } + + private func resolveSystemRunApproval( + req: BridgeInvokeRequest, + params: OpenClawSystemRunParams, + context: ExecRunContext) async -> ExecApprovalOutcome + { + let requiresAsk = ExecApprovalHelpers.requiresAsk( + ask: context.ask, + security: context.security, + allowlistMatch: context.allowlistMatch, + skillAllow: context.skillAllow) + + let decisionFromParams = ExecApprovalHelpers.parseDecision(params.approvalDecision) + var approvedByAsk = params.approved == true || decisionFromParams != nil + var persistAllowlist = decisionFromParams == .allowAlways + if decisionFromParams == .deny { + await self.emitExecEvent( + "exec.denied", + payload: ExecEventPayload( + sessionKey: context.sessionKey, + runId: context.runId, + host: "node", + command: context.displayCommand, + reason: "user-denied")) + return ExecApprovalOutcome( + approvedByAsk: approvedByAsk, + persistAllowlist: persistAllowlist, + response: Self.errorResponse( + req, + code: .unavailable, + message: "SYSTEM_RUN_DENIED: user denied")) + } + + if requiresAsk, !approvedByAsk { + let decision = await MainActor.run { + ExecApprovalsPromptPresenter.prompt( + ExecApprovalPromptRequest( + command: context.displayCommand, + cwd: params.cwd, + host: "node", + security: context.security.rawValue, + ask: context.ask.rawValue, + agentId: context.agentId, + resolvedPath: context.resolution?.resolvedPath, + sessionKey: context.sessionKey)) + } + switch decision { + case .deny: + await self.emitExecEvent( + "exec.denied", + payload: ExecEventPayload( + sessionKey: context.sessionKey, + runId: context.runId, + host: "node", + command: context.displayCommand, + reason: "user-denied")) + return ExecApprovalOutcome( + approvedByAsk: approvedByAsk, + persistAllowlist: persistAllowlist, + response: Self.errorResponse( + req, + code: .unavailable, + message: "SYSTEM_RUN_DENIED: user denied")) + case .allowAlways: + approvedByAsk = true + persistAllowlist = true + case .allowOnce: + approvedByAsk = true + } + } + + return ExecApprovalOutcome( + approvedByAsk: approvedByAsk, + persistAllowlist: persistAllowlist, + response: nil) + } + + private func handleSystemExecApprovalsGet(_ req: BridgeInvokeRequest) async throws -> BridgeInvokeResponse { + _ = ExecApprovalsStore.ensureFile() + let snapshot = ExecApprovalsStore.readSnapshot() + let redacted = ExecApprovalsSnapshot( + path: snapshot.path, + exists: snapshot.exists, + hash: snapshot.hash, + file: ExecApprovalsStore.redactForSnapshot(snapshot.file)) + let payload = try Self.encodePayload(redacted) + return BridgeInvokeResponse(id: req.id, ok: true, payloadJSON: payload) + } + + private func handleSystemExecApprovalsSet(_ req: BridgeInvokeRequest) async throws -> BridgeInvokeResponse { + struct SetParams: Decodable { + var file: ExecApprovalsFile + var baseHash: String? + } + + let params = try Self.decodeParams(SetParams.self, from: req.paramsJSON) + let current = ExecApprovalsStore.ensureFile() + let snapshot = ExecApprovalsStore.readSnapshot() + if snapshot.exists { + if snapshot.hash.isEmpty { + return Self.errorResponse( + req, + code: .invalidRequest, + message: "INVALID_REQUEST: exec approvals base hash unavailable; reload and retry") + } + let baseHash = params.baseHash?.trimmingCharacters(in: .whitespacesAndNewlines) ?? "" + if baseHash.isEmpty { + return Self.errorResponse( + req, + code: .invalidRequest, + message: "INVALID_REQUEST: exec approvals base hash required; reload and retry") + } + if baseHash != snapshot.hash { + return Self.errorResponse( + req, + code: .invalidRequest, + message: "INVALID_REQUEST: exec approvals changed; reload and retry") + } + } + + var normalized = ExecApprovalsStore.normalizeIncoming(params.file) + let socketPath = normalized.socket?.path?.trimmingCharacters(in: .whitespacesAndNewlines) + let token = normalized.socket?.token?.trimmingCharacters(in: .whitespacesAndNewlines) + let resolvedPath = (socketPath?.isEmpty == false) + ? socketPath! + : current.socket?.path?.trimmingCharacters(in: .whitespacesAndNewlines) ?? + ExecApprovalsStore.socketPath() + let resolvedToken = (token?.isEmpty == false) + ? token! + : current.socket?.token?.trimmingCharacters(in: .whitespacesAndNewlines) ?? "" + normalized.socket = ExecApprovalsSocketConfig(path: resolvedPath, token: resolvedToken) + + ExecApprovalsStore.saveFile(normalized) + let nextSnapshot = ExecApprovalsStore.readSnapshot() + let redacted = ExecApprovalsSnapshot( + path: nextSnapshot.path, + exists: nextSnapshot.exists, + hash: nextSnapshot.hash, + file: ExecApprovalsStore.redactForSnapshot(nextSnapshot.file)) + let payload = try Self.encodePayload(redacted) + return BridgeInvokeResponse(id: req.id, ok: true, payloadJSON: payload) + } + + private func emitExecEvent(_ event: String, payload: ExecEventPayload) async { + guard let sender = self.eventSender else { return } + guard let data = try? JSONEncoder().encode(payload), + let json = String(data: data, encoding: .utf8) + else { + return + } + await sender(event, json) + } + + private func handleSystemNotify(_ req: BridgeInvokeRequest) async throws -> BridgeInvokeResponse { + let params = try Self.decodeParams(OpenClawSystemNotifyParams.self, from: req.paramsJSON) + let title = params.title.trimmingCharacters(in: .whitespacesAndNewlines) + let body = params.body.trimmingCharacters(in: .whitespacesAndNewlines) + if title.isEmpty, body.isEmpty { + return Self.errorResponse(req, code: .invalidRequest, message: "INVALID_REQUEST: empty notification") + } + + let priority = params.priority.flatMap { NotificationPriority(rawValue: $0.rawValue) } + let delivery = params.delivery.flatMap { NotificationDelivery(rawValue: $0.rawValue) } ?? .system + let manager = NotificationManager() + + switch delivery { + case .system: + let ok = await manager.send( + title: title, + body: body, + sound: params.sound, + priority: priority) + return ok + ? BridgeInvokeResponse(id: req.id, ok: true) + : Self.errorResponse(req, code: .unavailable, message: "NOT_AUTHORIZED: notifications") + case .overlay: + await NotifyOverlayController.shared.present(title: title, body: body) + return BridgeInvokeResponse(id: req.id, ok: true) + case .auto: + let ok = await manager.send( + title: title, + body: body, + sound: params.sound, + priority: priority) + if ok { + return BridgeInvokeResponse(id: req.id, ok: true) + } + await NotifyOverlayController.shared.present(title: title, body: body) + return BridgeInvokeResponse(id: req.id, ok: true) + } + } +} + +extension MacNodeRuntime { + private static func decodeParams(_ type: T.Type, from json: String?) throws -> T { + guard let json, let data = json.data(using: .utf8) else { + throw NSError(domain: "Gateway", code: 20, userInfo: [ + NSLocalizedDescriptionKey: "INVALID_REQUEST: paramsJSON required", + ]) + } + return try JSONDecoder().decode(type, from: data) + } + + private static func encodePayload(_ obj: some Encodable) throws -> String { + let data = try JSONEncoder().encode(obj) + guard let json = String(bytes: data, encoding: .utf8) else { + throw NSError(domain: "Node", code: 21, userInfo: [ + NSLocalizedDescriptionKey: "Failed to encode payload as UTF-8", + ]) + } + return json + } + + private nonisolated static func canvasEnabled() -> Bool { + UserDefaults.standard.object(forKey: canvasEnabledKey) as? Bool ?? true + } + + private nonisolated static func cameraEnabled() -> Bool { + UserDefaults.standard.object(forKey: cameraEnabledKey) as? Bool ?? false + } + + private static let blockedEnvKeys: Set = [ + "PATH", + "NODE_OPTIONS", + "PYTHONHOME", + "PYTHONPATH", + "PERL5LIB", + "PERL5OPT", + "RUBYOPT", + ] + + private static let blockedEnvPrefixes: [String] = [ + "DYLD_", + "LD_", + ] + + private static func sanitizedEnv(_ overrides: [String: String]?) -> [String: String]? { + guard let overrides else { return nil } + var merged = ProcessInfo.processInfo.environment + for (rawKey, value) in overrides { + let key = rawKey.trimmingCharacters(in: .whitespacesAndNewlines) + guard !key.isEmpty else { continue } + let upper = key.uppercased() + if self.blockedEnvKeys.contains(upper) { continue } + if self.blockedEnvPrefixes.contains(where: { upper.hasPrefix($0) }) { continue } + merged[key] = value + } + return merged + } + + private nonisolated static func locationMode() -> OpenClawLocationMode { + let raw = UserDefaults.standard.string(forKey: locationModeKey) ?? "off" + return OpenClawLocationMode(rawValue: raw) ?? .off + } + + private nonisolated static func locationPreciseEnabled() -> Bool { + if UserDefaults.standard.object(forKey: locationPreciseKey) == nil { return true } + return UserDefaults.standard.bool(forKey: locationPreciseKey) + } + + private static func errorResponse( + _ req: BridgeInvokeRequest, + code: OpenClawNodeErrorCode, + message: String) -> BridgeInvokeResponse + { + BridgeInvokeResponse( + id: req.id, + ok: false, + error: OpenClawNodeError(code: code, message: message)) + } + + private static func encodeCanvasSnapshot( + image: NSImage, + format: OpenClawCanvasSnapshotFormat, + maxWidth: Int?, + quality: Double) throws -> Data + { + let source = Self.scaleImage(image, maxWidth: maxWidth) ?? image + guard let tiff = source.tiffRepresentation, + let rep = NSBitmapImageRep(data: tiff) + else { + throw NSError(domain: "Canvas", code: 22, userInfo: [ + NSLocalizedDescriptionKey: "snapshot encode failed", + ]) + } + + switch format { + case .png: + guard let data = rep.representation(using: .png, properties: [:]) else { + throw NSError(domain: "Canvas", code: 23, userInfo: [ + NSLocalizedDescriptionKey: "png encode failed", + ]) + } + return data + case .jpeg: + let clamped = min(1.0, max(0.05, quality)) + guard let data = rep.representation( + using: .jpeg, + properties: [.compressionFactor: clamped]) + else { + throw NSError(domain: "Canvas", code: 24, userInfo: [ + NSLocalizedDescriptionKey: "jpeg encode failed", + ]) + } + return data + } + } + + private static func scaleImage(_ image: NSImage, maxWidth: Int?) -> NSImage? { + guard let maxWidth, maxWidth > 0 else { return image } + let size = image.size + guard size.width > 0, size.width > CGFloat(maxWidth) else { return image } + let scale = CGFloat(maxWidth) / size.width + let target = NSSize(width: CGFloat(maxWidth), height: size.height * scale) + + let out = NSImage(size: target) + out.lockFocus() + image.draw( + in: NSRect(origin: .zero, size: target), + from: NSRect(origin: .zero, size: size), + operation: .copy, + fraction: 1.0) + out.unlockFocus() + return out + } +} diff --git a/apps/macos/Sources/OpenClaw/NodeMode/MacNodeRuntimeMainActorServices.swift b/apps/macos/Sources/OpenClaw/NodeMode/MacNodeRuntimeMainActorServices.swift new file mode 100644 index 0000000000000000000000000000000000000000..982ec8bf90f9e4e4d2e599910291c64939fe829b --- /dev/null +++ b/apps/macos/Sources/OpenClaw/NodeMode/MacNodeRuntimeMainActorServices.swift @@ -0,0 +1,60 @@ +import OpenClawKit +import CoreLocation +import Foundation + +@MainActor +protocol MacNodeRuntimeMainActorServices: Sendable { + func recordScreen( + screenIndex: Int?, + durationMs: Int?, + fps: Double?, + includeAudio: Bool?, + outPath: String?) async throws -> (path: String, hasAudio: Bool) + + func locationAuthorizationStatus() -> CLAuthorizationStatus + func locationAccuracyAuthorization() -> CLAccuracyAuthorization + func currentLocation( + desiredAccuracy: OpenClawLocationAccuracy, + maxAgeMs: Int?, + timeoutMs: Int?) async throws -> CLLocation +} + +@MainActor +final class LiveMacNodeRuntimeMainActorServices: MacNodeRuntimeMainActorServices, @unchecked Sendable { + private let screenRecorder = ScreenRecordService() + private let locationService = MacNodeLocationService() + + func recordScreen( + screenIndex: Int?, + durationMs: Int?, + fps: Double?, + includeAudio: Bool?, + outPath: String?) async throws -> (path: String, hasAudio: Bool) + { + try await self.screenRecorder.record( + screenIndex: screenIndex, + durationMs: durationMs, + fps: fps, + includeAudio: includeAudio, + outPath: outPath) + } + + func locationAuthorizationStatus() -> CLAuthorizationStatus { + self.locationService.authorizationStatus() + } + + func locationAccuracyAuthorization() -> CLAccuracyAuthorization { + self.locationService.accuracyAuthorization() + } + + func currentLocation( + desiredAccuracy: OpenClawLocationAccuracy, + maxAgeMs: Int?, + timeoutMs: Int?) async throws -> CLLocation + { + try await self.locationService.currentLocation( + desiredAccuracy: desiredAccuracy, + maxAgeMs: maxAgeMs, + timeoutMs: timeoutMs) + } +} diff --git a/apps/macos/Sources/OpenClaw/NodeMode/MacNodeScreenCommands.swift b/apps/macos/Sources/OpenClaw/NodeMode/MacNodeScreenCommands.swift new file mode 100644 index 0000000000000000000000000000000000000000..6f849fdf03adc24b6d5fd92ea5d615ebff40b6dd --- /dev/null +++ b/apps/macos/Sources/OpenClaw/NodeMode/MacNodeScreenCommands.swift @@ -0,0 +1,13 @@ +import Foundation + +enum MacNodeScreenCommand: String, Codable, Sendable { + case record = "screen.record" +} + +struct MacNodeScreenRecordParams: Codable, Sendable, Equatable { + var screenIndex: Int? + var durationMs: Int? + var fps: Double? + var format: String? + var includeAudio: Bool? +} diff --git a/apps/macos/Sources/OpenClaw/NodePairingApprovalPrompter.swift b/apps/macos/Sources/OpenClaw/NodePairingApprovalPrompter.swift new file mode 100644 index 0000000000000000000000000000000000000000..9853294662432022b1af85be52b805dd891e8f30 --- /dev/null +++ b/apps/macos/Sources/OpenClaw/NodePairingApprovalPrompter.swift @@ -0,0 +1,708 @@ +import AppKit +import OpenClawDiscovery +import OpenClawIPC +import OpenClawKit +import OpenClawProtocol +import Foundation +import Observation +import OSLog +import UserNotifications + +enum NodePairingReconcilePolicy { + static let activeIntervalMs: UInt64 = 15000 + static let resyncDelayMs: UInt64 = 250 + + static func shouldPoll(pendingCount: Int, isPresenting: Bool) -> Bool { + pendingCount > 0 || isPresenting + } +} + +@MainActor +@Observable +final class NodePairingApprovalPrompter { + static let shared = NodePairingApprovalPrompter() + + private let logger = Logger(subsystem: "ai.openclaw", category: "node-pairing") + private var task: Task? + private var reconcileTask: Task? + private var reconcileOnceTask: Task? + private var reconcileInFlight = false + private var isStopping = false + private var isPresenting = false + private var queue: [PendingRequest] = [] + var pendingCount: Int = 0 + var pendingRepairCount: Int = 0 + private var activeAlert: NSAlert? + private var activeRequestId: String? + private var alertHostWindow: NSWindow? + private var remoteResolutionsByRequestId: [String: PairingResolution] = [:] + private var autoApproveAttempts: Set = [] + + private final class AlertHostWindow: NSWindow { + override var canBecomeKey: Bool { true } + override var canBecomeMain: Bool { true } + } + + private struct PairingList: Codable { + let pending: [PendingRequest] + let paired: [PairedNode]? + } + + private struct PairedNode: Codable, Equatable { + let nodeId: String + let approvedAtMs: Double? + let displayName: String? + let platform: String? + let version: String? + let remoteIp: String? + } + + private struct PendingRequest: Codable, Equatable, Identifiable { + let requestId: String + let nodeId: String + let displayName: String? + let platform: String? + let version: String? + let remoteIp: String? + let isRepair: Bool? + let silent: Bool? + let ts: Double + + var id: String { self.requestId } + } + + private struct PairingResolvedEvent: Codable { + let requestId: String + let nodeId: String + let decision: String + let ts: Double + } + + private enum PairingResolution: String { + case approved + case rejected + } + + func start() { + guard self.task == nil else { return } + self.isStopping = false + self.reconcileTask?.cancel() + self.reconcileTask = nil + self.task = Task { [weak self] in + guard let self else { return } + _ = try? await GatewayConnection.shared.refresh() + await self.loadPendingRequestsFromGateway() + let stream = await GatewayConnection.shared.subscribe(bufferingNewest: 200) + for await push in stream { + if Task.isCancelled { return } + await MainActor.run { [weak self] in self?.handle(push: push) } + } + } + } + + func stop() { + self.isStopping = true + self.endActiveAlert() + self.task?.cancel() + self.task = nil + self.reconcileTask?.cancel() + self.reconcileTask = nil + self.reconcileOnceTask?.cancel() + self.reconcileOnceTask = nil + self.queue.removeAll(keepingCapacity: false) + self.updatePendingCounts() + self.isPresenting = false + self.activeRequestId = nil + self.alertHostWindow?.orderOut(nil) + self.alertHostWindow?.close() + self.alertHostWindow = nil + self.remoteResolutionsByRequestId.removeAll(keepingCapacity: false) + self.autoApproveAttempts.removeAll(keepingCapacity: false) + } + + private func loadPendingRequestsFromGateway() async { + // The gateway process may start slightly after the app. Retry a bit so + // pending pairing prompts are still shown on launch. + var delayMs: UInt64 = 200 + for attempt in 1...8 { + if Task.isCancelled { return } + do { + let data = try await GatewayConnection.shared.request( + method: "node.pair.list", + params: nil, + timeoutMs: 6000) + guard !data.isEmpty else { return } + let list = try JSONDecoder().decode(PairingList.self, from: data) + let pendingCount = list.pending.count + guard pendingCount > 0 else { return } + self.logger.info( + "loaded \(pendingCount, privacy: .public) pending node pairing request(s) on startup") + await self.apply(list: list) + return + } catch { + if attempt == 8 { + self.logger + .error( + "failed to load pending pairing requests: \(error.localizedDescription, privacy: .public)") + return + } + try? await Task.sleep(nanoseconds: delayMs * 1_000_000) + delayMs = min(delayMs * 2, 2000) + } + } + } + + private func reconcileLoop() async { + // Reconcile requests periodically so multiple running apps stay in sync + // (e.g. close dialogs + notify if another machine approves/rejects via app or CLI). + while !Task.isCancelled { + if self.isStopping { break } + if !self.shouldPoll { + self.reconcileTask = nil + return + } + await self.reconcileOnce(timeoutMs: 2500) + try? await Task.sleep( + nanoseconds: NodePairingReconcilePolicy.activeIntervalMs * 1_000_000) + } + self.reconcileTask = nil + } + + private func fetchPairingList(timeoutMs: Double) async throws -> PairingList { + let data = try await GatewayConnection.shared.request( + method: "node.pair.list", + params: nil, + timeoutMs: timeoutMs) + return try JSONDecoder().decode(PairingList.self, from: data) + } + + private func apply(list: PairingList) async { + if self.isStopping { return } + + let pendingById = Dictionary( + uniqueKeysWithValues: list.pending.map { ($0.requestId, $0) }) + + // Enqueue any missing requests (covers missed pushes while reconnecting). + for req in list.pending.sorted(by: { $0.ts < $1.ts }) { + self.enqueue(req) + } + + // Detect resolved requests (approved/rejected elsewhere). + let queued = self.queue + for req in queued { + if pendingById[req.requestId] != nil { continue } + let resolution = self.inferResolution(for: req, list: list) + + if self.activeRequestId == req.requestId, self.activeAlert != nil { + self.remoteResolutionsByRequestId[req.requestId] = resolution + self.logger.info( + """ + pairing request resolved elsewhere; closing dialog \ + requestId=\(req.requestId, privacy: .public) \ + resolution=\(resolution.rawValue, privacy: .public) + """) + self.endActiveAlert() + continue + } + + self.logger.info( + """ + pairing request resolved elsewhere requestId=\(req.requestId, privacy: .public) \ + resolution=\(resolution.rawValue, privacy: .public) + """) + self.queue.removeAll { $0 == req } + Task { @MainActor in + await self.notify(resolution: resolution, request: req, via: "remote") + } + } + + if self.queue.isEmpty { + self.isPresenting = false + } + self.presentNextIfNeeded() + self.updateReconcileLoop() + } + + private func inferResolution(for request: PendingRequest, list: PairingList) -> PairingResolution { + let paired = list.paired ?? [] + guard let node = paired.first(where: { $0.nodeId == request.nodeId }) else { + return .rejected + } + if request.isRepair == true, let approvedAtMs = node.approvedAtMs { + return approvedAtMs >= request.ts ? .approved : .rejected + } + return .approved + } + + private func endActiveAlert() { + guard let alert = self.activeAlert else { return } + if let parent = alert.window.sheetParent { + parent.endSheet(alert.window, returnCode: .abort) + } + self.activeAlert = nil + self.activeRequestId = nil + } + + private func requireAlertHostWindow() -> NSWindow { + if let alertHostWindow { + return alertHostWindow + } + + let window = AlertHostWindow( + contentRect: NSRect(x: 0, y: 0, width: 520, height: 1), + styleMask: [.borderless], + backing: .buffered, + defer: false) + window.title = "" + window.isReleasedWhenClosed = false + window.level = .floating + window.collectionBehavior = [.canJoinAllSpaces, .fullScreenAuxiliary] + window.isOpaque = false + window.hasShadow = false + window.backgroundColor = .clear + window.ignoresMouseEvents = true + + self.alertHostWindow = window + return window + } + + private func handle(push: GatewayPush) { + switch push { + case let .event(evt) where evt.event == "node.pair.requested": + guard let payload = evt.payload else { return } + do { + let req = try GatewayPayloadDecoding.decode(payload, as: PendingRequest.self) + self.enqueue(req) + } catch { + self.logger + .error("failed to decode pairing request: \(error.localizedDescription, privacy: .public)") + } + case let .event(evt) where evt.event == "node.pair.resolved": + guard let payload = evt.payload else { return } + do { + let resolved = try GatewayPayloadDecoding.decode(payload, as: PairingResolvedEvent.self) + self.handleResolved(resolved) + } catch { + self.logger + .error( + "failed to decode pairing resolution: \(error.localizedDescription, privacy: .public)") + } + case .snapshot: + self.scheduleReconcileOnce(delayMs: 0) + case .seqGap: + self.scheduleReconcileOnce() + default: + return + } + } + + private func enqueue(_ req: PendingRequest) { + if self.queue.contains(req) { return } + self.queue.append(req) + self.updatePendingCounts() + self.presentNextIfNeeded() + self.updateReconcileLoop() + } + + private func presentNextIfNeeded() { + guard !self.isStopping else { return } + guard !self.isPresenting else { return } + guard let next = self.queue.first else { return } + self.isPresenting = true + Task { @MainActor [weak self] in + guard let self else { return } + if await self.trySilentApproveIfPossible(next) { + return + } + self.presentAlert(for: next) + } + } + + private func presentAlert(for req: PendingRequest) { + self.logger.info("presenting node pairing alert requestId=\(req.requestId, privacy: .public)") + NSApp.activate(ignoringOtherApps: true) + + let alert = NSAlert() + alert.alertStyle = .warning + alert.messageText = "Allow node to connect?" + alert.informativeText = Self.describe(req) + // Fail-safe ordering: if the dialog can't be presented, default to "Later". + alert.addButton(withTitle: "Later") + alert.addButton(withTitle: "Approve") + alert.addButton(withTitle: "Reject") + if #available(macOS 11.0, *), alert.buttons.indices.contains(2) { + alert.buttons[2].hasDestructiveAction = true + } + + self.activeAlert = alert + self.activeRequestId = req.requestId + let hostWindow = self.requireAlertHostWindow() + + // Position the hidden host window so the sheet appears centered on screen. + // (Sheets attach to the top edge of their parent window; if the parent is tiny, it looks "anchored".) + let sheetSize = alert.window.frame.size + if let screen = hostWindow.screen ?? NSScreen.main { + let bounds = screen.visibleFrame + let x = bounds.midX - (sheetSize.width / 2) + let sheetOriginY = bounds.midY - (sheetSize.height / 2) + let hostY = sheetOriginY + sheetSize.height - hostWindow.frame.height + hostWindow.setFrameOrigin(NSPoint(x: x, y: hostY)) + } else { + hostWindow.center() + } + + hostWindow.makeKeyAndOrderFront(nil) + alert.beginSheetModal(for: hostWindow) { [weak self] response in + Task { @MainActor [weak self] in + guard let self else { return } + self.activeRequestId = nil + self.activeAlert = nil + await self.handleAlertResponse(response, request: req) + hostWindow.orderOut(nil) + } + } + } + + private func handleAlertResponse(_ response: NSApplication.ModalResponse, request: PendingRequest) async { + defer { + if self.queue.first == request { + self.queue.removeFirst() + } else { + self.queue.removeAll { $0 == request } + } + self.updatePendingCounts() + self.isPresenting = false + self.presentNextIfNeeded() + self.updateReconcileLoop() + } + + // Never approve/reject while shutting down (alerts can get dismissed during app termination). + guard !self.isStopping else { return } + + if let resolved = self.remoteResolutionsByRequestId.removeValue(forKey: request.requestId) { + await self.notify(resolution: resolved, request: request, via: "remote") + return + } + + switch response { + case .alertFirstButtonReturn: + // Later: leave as pending (CLI can approve/reject). Request will expire on the gateway TTL. + return + case .alertSecondButtonReturn: + _ = await self.approve(requestId: request.requestId) + await self.notify(resolution: .approved, request: request, via: "local") + case .alertThirdButtonReturn: + await self.reject(requestId: request.requestId) + await self.notify(resolution: .rejected, request: request, via: "local") + default: + return + } + } + + private func approve(requestId: String) async -> Bool { + do { + try await GatewayConnection.shared.nodePairApprove(requestId: requestId) + self.logger.info("approved node pairing requestId=\(requestId, privacy: .public)") + return true + } catch { + self.logger.error("approve failed requestId=\(requestId, privacy: .public)") + self.logger.error("approve failed: \(error.localizedDescription, privacy: .public)") + return false + } + } + + private func reject(requestId: String) async { + do { + try await GatewayConnection.shared.nodePairReject(requestId: requestId) + self.logger.info("rejected node pairing requestId=\(requestId, privacy: .public)") + } catch { + self.logger.error("reject failed requestId=\(requestId, privacy: .public)") + self.logger.error("reject failed: \(error.localizedDescription, privacy: .public)") + } + } + + private static func describe(_ req: PendingRequest) -> String { + let name = req.displayName?.trimmingCharacters(in: .whitespacesAndNewlines) + let platform = self.prettyPlatform(req.platform) + let version = req.version?.trimmingCharacters(in: .whitespacesAndNewlines) + let ip = self.prettyIP(req.remoteIp) + + var lines: [String] = [] + lines.append("Name: \(name?.isEmpty == false ? name! : "Unknown")") + lines.append("Node ID: \(req.nodeId)") + if let platform, !platform.isEmpty { lines.append("Platform: \(platform)") } + if let version, !version.isEmpty { lines.append("App: \(version)") } + if let ip, !ip.isEmpty { lines.append("IP: \(ip)") } + if req.isRepair == true { lines.append("Note: Repair request (token will rotate).") } + return lines.joined(separator: "\n") + } + + private static func prettyIP(_ ip: String?) -> String? { + let trimmed = ip?.trimmingCharacters(in: .whitespacesAndNewlines) + guard let trimmed, !trimmed.isEmpty else { return nil } + return trimmed.replacingOccurrences(of: "::ffff:", with: "") + } + + private static func prettyPlatform(_ platform: String?) -> String? { + let raw = platform?.trimmingCharacters(in: .whitespacesAndNewlines) + guard let raw, !raw.isEmpty else { return nil } + if raw.lowercased() == "ios" { return "iOS" } + if raw.lowercased() == "macos" { return "macOS" } + return raw + } + + private func notify(resolution: PairingResolution, request: PendingRequest, via: String) async { + let center = UNUserNotificationCenter.current() + let settings = await center.notificationSettings() + guard settings.authorizationStatus == .authorized || + settings.authorizationStatus == .provisional + else { + return + } + + let title = resolution == .approved ? "Node pairing approved" : "Node pairing rejected" + let name = request.displayName?.trimmingCharacters(in: .whitespacesAndNewlines) + let device = name?.isEmpty == false ? name! : request.nodeId + let body = "\(device)\n(via \(via))" + + _ = await NotificationManager().send( + title: title, + body: body, + sound: nil, + priority: .active) + } + + private struct SSHTarget { + let host: String + let port: Int + } + + private func trySilentApproveIfPossible(_ req: PendingRequest) async -> Bool { + guard req.silent == true else { return false } + if self.autoApproveAttempts.contains(req.requestId) { return false } + self.autoApproveAttempts.insert(req.requestId) + + guard let target = await self.resolveSSHTarget() else { + self.logger.info("silent pairing skipped (no ssh target) requestId=\(req.requestId, privacy: .public)") + return false + } + + let user = NSUserName().trimmingCharacters(in: .whitespacesAndNewlines) + guard !user.isEmpty else { + self.logger.info("silent pairing skipped (missing local user) requestId=\(req.requestId, privacy: .public)") + return false + } + + let ok = await Self.probeSSH(user: user, host: target.host, port: target.port) + if !ok { + self.logger.info("silent pairing probe failed requestId=\(req.requestId, privacy: .public)") + return false + } + + guard await self.approve(requestId: req.requestId) else { + self.logger.info("silent pairing approve failed requestId=\(req.requestId, privacy: .public)") + return false + } + + await self.notify(resolution: .approved, request: req, via: "silent-ssh") + if self.queue.first == req { + self.queue.removeFirst() + } else { + self.queue.removeAll { $0 == req } + } + + self.updatePendingCounts() + self.isPresenting = false + self.presentNextIfNeeded() + self.updateReconcileLoop() + return true + } + + private func resolveSSHTarget() async -> SSHTarget? { + let settings = CommandResolver.connectionSettings() + if !settings.target.isEmpty, let parsed = CommandResolver.parseSSHTarget(settings.target) { + let user = NSUserName().trimmingCharacters(in: .whitespacesAndNewlines) + if let targetUser = parsed.user, + !targetUser.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty, + targetUser != user + { + self.logger.info("silent pairing skipped (ssh user mismatch)") + return nil + } + let host = parsed.host.trimmingCharacters(in: .whitespacesAndNewlines) + guard !host.isEmpty else { return nil } + let port = parsed.port > 0 ? parsed.port : 22 + return SSHTarget(host: host, port: port) + } + + let model = GatewayDiscoveryModel(localDisplayName: InstanceIdentity.displayName) + model.start() + defer { model.stop() } + + let deadline = Date().addingTimeInterval(5.0) + while model.gateways.isEmpty, Date() < deadline { + try? await Task.sleep(nanoseconds: 200_000_000) + } + + let preferred = GatewayDiscoveryPreferences.preferredStableID() + let gateway = model.gateways.first { $0.stableID == preferred } ?? model.gateways.first + guard let gateway else { return nil } + let host = (gateway.tailnetDns?.trimmingCharacters(in: .whitespacesAndNewlines).nonEmpty ?? + gateway.lanHost?.trimmingCharacters(in: .whitespacesAndNewlines).nonEmpty) + guard let host, !host.isEmpty else { return nil } + let port = gateway.sshPort > 0 ? gateway.sshPort : 22 + return SSHTarget(host: host, port: port) + } + + private static func probeSSH(user: String, host: String, port: Int) async -> Bool { + await Task.detached(priority: .utility) { + let process = Process() + process.executableURL = URL(fileURLWithPath: "/usr/bin/ssh") + + let options = [ + "-o", "BatchMode=yes", + "-o", "ConnectTimeout=5", + "-o", "NumberOfPasswordPrompts=0", + "-o", "PreferredAuthentications=publickey", + "-o", "StrictHostKeyChecking=accept-new", + ] + guard let target = CommandResolver.makeSSHTarget(user: user, host: host, port: port) else { + return false + } + let args = CommandResolver.sshArguments( + target: target, + identity: "", + options: options, + remoteCommand: ["/usr/bin/true"]) + process.arguments = args + let pipe = Pipe() + process.standardOutput = pipe + process.standardError = pipe + + do { + _ = try process.runAndReadToEnd(from: pipe) + } catch { + return false + } + return process.terminationStatus == 0 + }.value + } + + private var shouldPoll: Bool { + NodePairingReconcilePolicy.shouldPoll( + pendingCount: self.queue.count, + isPresenting: self.isPresenting) + } + + private func updateReconcileLoop() { + guard !self.isStopping else { return } + if self.shouldPoll { + if self.reconcileTask == nil { + self.reconcileTask = Task { [weak self] in + await self?.reconcileLoop() + } + } + } else { + self.reconcileTask?.cancel() + self.reconcileTask = nil + } + } + + private func updatePendingCounts() { + // Keep a cheap observable summary for the menu bar status line. + self.pendingCount = self.queue.count + self.pendingRepairCount = self.queue.count(where: { $0.isRepair == true }) + } + + private func reconcileOnce(timeoutMs: Double) async { + if self.isStopping { return } + if self.reconcileInFlight { return } + self.reconcileInFlight = true + defer { self.reconcileInFlight = false } + do { + let list = try await self.fetchPairingList(timeoutMs: timeoutMs) + await self.apply(list: list) + } catch { + // best effort: ignore transient connectivity failures + } + } + + private func scheduleReconcileOnce(delayMs: UInt64 = NodePairingReconcilePolicy.resyncDelayMs) { + self.reconcileOnceTask?.cancel() + self.reconcileOnceTask = Task { [weak self] in + guard let self else { return } + if delayMs > 0 { + try? await Task.sleep(nanoseconds: delayMs * 1_000_000) + } + await self.reconcileOnce(timeoutMs: 2500) + } + } + + private func handleResolved(_ resolved: PairingResolvedEvent) { + let resolution: PairingResolution = + resolved.decision == PairingResolution.approved.rawValue ? .approved : .rejected + + if self.activeRequestId == resolved.requestId, self.activeAlert != nil { + self.remoteResolutionsByRequestId[resolved.requestId] = resolution + self.logger.info( + """ + pairing request resolved elsewhere; closing dialog \ + requestId=\(resolved.requestId, privacy: .public) \ + resolution=\(resolution.rawValue, privacy: .public) + """) + self.endActiveAlert() + return + } + + guard let request = self.queue.first(where: { $0.requestId == resolved.requestId }) else { + return + } + self.queue.removeAll { $0.requestId == resolved.requestId } + self.updatePendingCounts() + Task { @MainActor in + await self.notify(resolution: resolution, request: request, via: "remote") + } + if self.queue.isEmpty { + self.isPresenting = false + } + self.presentNextIfNeeded() + self.updateReconcileLoop() + } +} + +#if DEBUG +@MainActor +extension NodePairingApprovalPrompter { + static func exerciseForTesting() async { + let prompter = NodePairingApprovalPrompter() + let pending = PendingRequest( + requestId: "req-1", + nodeId: "node-1", + displayName: "Node One", + platform: "macos", + version: "1.0.0", + remoteIp: "127.0.0.1", + isRepair: false, + silent: true, + ts: 1_700_000_000_000) + let paired = PairedNode( + nodeId: "node-1", + approvedAtMs: 1_700_000_000_000, + displayName: "Node One", + platform: "macOS", + version: "1.0.0", + remoteIp: "127.0.0.1") + let list = PairingList(pending: [pending], paired: [paired]) + + _ = Self.describe(pending) + _ = Self.prettyIP(pending.remoteIp) + _ = Self.prettyPlatform(pending.platform) + _ = prompter.inferResolution(for: pending, list: list) + + prompter.queue = [pending] + _ = prompter.shouldPoll + _ = await prompter.trySilentApproveIfPossible(pending) + prompter.queue.removeAll() + } +} +#endif diff --git a/apps/macos/Sources/OpenClaw/NodeServiceManager.swift b/apps/macos/Sources/OpenClaw/NodeServiceManager.swift new file mode 100644 index 0000000000000000000000000000000000000000..38d0aa30241bf86c869508ef4289bd4c0fafdccf --- /dev/null +++ b/apps/macos/Sources/OpenClaw/NodeServiceManager.swift @@ -0,0 +1,150 @@ +import Foundation +import OSLog + +enum NodeServiceManager { + private static let logger = Logger(subsystem: "ai.openclaw", category: "node.service") + + static func start() async -> String? { + let result = await self.runServiceCommandResult( + ["node", "start"], + timeout: 20, + quiet: false) + if let error = self.errorMessage(from: result, treatNotLoadedAsError: true) { + self.logger.error("node service start failed: \(error, privacy: .public)") + return error + } + return nil + } + + static func stop() async -> String? { + let result = await self.runServiceCommandResult( + ["node", "stop"], + timeout: 15, + quiet: false) + if let error = self.errorMessage(from: result, treatNotLoadedAsError: false) { + self.logger.error("node service stop failed: \(error, privacy: .public)") + return error + } + return nil + } +} + +extension NodeServiceManager { + private struct CommandResult { + let success: Bool + let payload: Data? + let message: String? + let parsed: ParsedServiceJson? + } + + private struct ParsedServiceJson { + let text: String + let object: [String: Any] + let ok: Bool? + let result: String? + let message: String? + let error: String? + let hints: [String] + } + + private static func runServiceCommandResult( + _ args: [String], + timeout: Double, + quiet: Bool) async -> CommandResult + { + let command = CommandResolver.openclawCommand( + subcommand: "service", + extraArgs: self.withJsonFlag(args), + // Service management must always run locally, even if remote mode is configured. + configRoot: ["gateway": ["mode": "local"]]) + var env = ProcessInfo.processInfo.environment + env["PATH"] = CommandResolver.preferredPaths().joined(separator: ":") + let response = await ShellExecutor.runDetailed(command: command, cwd: nil, env: env, timeout: timeout) + let parsed = self.parseServiceJson(from: response.stdout) ?? self.parseServiceJson(from: response.stderr) + let ok = parsed?.ok + let message = parsed?.error ?? parsed?.message + let payload = parsed?.text.data(using: .utf8) + ?? (response.stdout.isEmpty ? response.stderr : response.stdout).data(using: .utf8) + let success = ok ?? response.success + if success { + return CommandResult(success: true, payload: payload, message: nil, parsed: parsed) + } + + if quiet { + return CommandResult(success: false, payload: payload, message: message, parsed: parsed) + } + + let detail = message ?? self.summarize(response.stderr) ?? self.summarize(response.stdout) + let exit = response.exitCode.map { "exit \($0)" } ?? (response.errorMessage ?? "failed") + let fullMessage = detail.map { "Node service command failed (\(exit)): \($0)" } + ?? "Node service command failed (\(exit))" + self.logger.error("\(fullMessage, privacy: .public)") + return CommandResult(success: false, payload: payload, message: detail, parsed: parsed) + } + + private static func errorMessage(from result: CommandResult, treatNotLoadedAsError: Bool) -> String? { + if !result.success { + return result.message ?? "Node service command failed" + } + guard let parsed = result.parsed else { return nil } + if parsed.ok == false { + return self.mergeHints(message: parsed.error ?? parsed.message, hints: parsed.hints) + } + if treatNotLoadedAsError, parsed.result == "not-loaded" { + let base = parsed.message ?? "Node service not loaded." + return self.mergeHints(message: base, hints: parsed.hints) + } + return nil + } + + private static func withJsonFlag(_ args: [String]) -> [String] { + if args.contains("--json") { return args } + return args + ["--json"] + } + + private static func parseServiceJson(from raw: String) -> ParsedServiceJson? { + let trimmed = raw.trimmingCharacters(in: .whitespacesAndNewlines) + guard let start = trimmed.firstIndex(of: "{"), + let end = trimmed.lastIndex(of: "}") + else { + return nil + } + let jsonText = String(trimmed[start...end]) + guard let data = jsonText.data(using: .utf8) else { return nil } + guard let object = try? JSONSerialization.jsonObject(with: data) as? [String: Any] else { return nil } + let ok = object["ok"] as? Bool + let result = object["result"] as? String + let message = object["message"] as? String + let error = object["error"] as? String + let hints = (object["hints"] as? [String]) ?? [] + return ParsedServiceJson( + text: jsonText, + object: object, + ok: ok, + result: result, + message: message, + error: error, + hints: hints) + } + + private static func mergeHints(message: String?, hints: [String]) -> String? { + let trimmed = message?.trimmingCharacters(in: .whitespacesAndNewlines) + let nonEmpty = trimmed?.isEmpty == false ? trimmed : nil + guard !hints.isEmpty else { return nonEmpty } + let hintText = hints.prefix(2).joined(separator: " · ") + if let nonEmpty { + return "\(nonEmpty) (\(hintText))" + } + return hintText + } + + private static func summarize(_ text: String) -> String? { + let lines = text + .split(whereSeparator: \.isNewline) + .map { $0.trimmingCharacters(in: .whitespacesAndNewlines) } + .filter { !$0.isEmpty } + guard let last = lines.last else { return nil } + let normalized = last.replacingOccurrences(of: "\\s+", with: " ", options: .regularExpression) + return normalized.count > 200 ? String(normalized.prefix(199)) + "…" : normalized + } +} diff --git a/apps/macos/Sources/OpenClaw/NodesMenu.swift b/apps/macos/Sources/OpenClaw/NodesMenu.swift new file mode 100644 index 0000000000000000000000000000000000000000..f88177d8dd02aad63f65241ab439a1e2e3a3f626 --- /dev/null +++ b/apps/macos/Sources/OpenClaw/NodesMenu.swift @@ -0,0 +1,333 @@ +import AppKit +import SwiftUI + +struct NodeMenuEntryFormatter { + static func isGateway(_ entry: NodeInfo) -> Bool { + entry.nodeId == "gateway" + } + + static func isConnected(_ entry: NodeInfo) -> Bool { + entry.isConnected + } + + static func primaryName(_ entry: NodeInfo) -> String { + if self.isGateway(entry) { + return entry.displayName?.nonEmpty ?? "Gateway" + } + return entry.displayName?.nonEmpty ?? entry.nodeId + } + + static func summaryText(_ entry: NodeInfo) -> String { + if self.isGateway(entry) { + let role = self.roleText(entry) + let name = self.primaryName(entry) + var parts = ["\(name) · \(role)"] + if let ip = entry.remoteIp?.nonEmpty { parts.append("host \(ip)") } + if let platform = self.platformText(entry) { parts.append(platform) } + return parts.joined(separator: " · ") + } + let name = self.primaryName(entry) + var prefix = "Node: \(name)" + if let ip = entry.remoteIp?.nonEmpty { + prefix += " (\(ip))" + } + var parts = [prefix] + if let platform = self.platformText(entry) { + parts.append("platform \(platform)") + } + let versionLabels = self.versionLabels(entry) + if !versionLabels.isEmpty { + parts.append(versionLabels.joined(separator: " · ")) + } + parts.append("status \(self.roleText(entry))") + return parts.joined(separator: " · ") + } + + static func roleText(_ entry: NodeInfo) -> String { + if entry.isConnected { return "connected" } + if self.isGateway(entry) { return "disconnected" } + if entry.isPaired { return "paired" } + return "unpaired" + } + + static func detailLeft(_ entry: NodeInfo) -> String { + let role = self.roleText(entry) + if let ip = entry.remoteIp?.nonEmpty { return "\(ip) · \(role)" } + return role + } + + static func headlineRight(_ entry: NodeInfo) -> String? { + self.platformText(entry) + } + + static func detailRightVersion(_ entry: NodeInfo) -> String? { + let labels = self.versionLabels(entry, compact: false) + if labels.isEmpty { return nil } + return labels.joined(separator: " · ") + } + + static func platformText(_ entry: NodeInfo) -> String? { + if let raw = entry.platform?.nonEmpty { + return self.prettyPlatform(raw) ?? raw + } + if let family = entry.deviceFamily?.lowercased() { + if family.contains("mac") { return "macOS" } + if family.contains("iphone") { return "iOS" } + if family.contains("ipad") { return "iPadOS" } + if family.contains("android") { return "Android" } + } + return nil + } + + private static func prettyPlatform(_ raw: String) -> String? { + let (prefix, version) = self.parsePlatform(raw) + if prefix.isEmpty { return nil } + let name: String = switch prefix { + case "macos": "macOS" + case "ios": "iOS" + case "ipados": "iPadOS" + case "tvos": "tvOS" + case "watchos": "watchOS" + default: prefix.prefix(1).uppercased() + prefix.dropFirst() + } + guard let version, !version.isEmpty else { return name } + let parts = version.split(separator: ".").map(String.init) + if parts.count >= 2 { + return "\(name) \(parts[0]).\(parts[1])" + } + return "\(name) \(version)" + } + + private static func parsePlatform(_ raw: String) -> (prefix: String, version: String?) { + let trimmed = raw.trimmingCharacters(in: .whitespacesAndNewlines) + if trimmed.isEmpty { return ("", nil) } + let parts = trimmed.split(whereSeparator: { $0 == " " || $0 == "\t" }).map(String.init) + let prefix = parts.first?.lowercased() ?? "" + let versionToken = parts.dropFirst().first + return (prefix, versionToken) + } + + private static func compactVersion(_ raw: String) -> String { + let trimmed = raw.trimmingCharacters(in: .whitespacesAndNewlines) + guard !trimmed.isEmpty else { return trimmed } + if let range = trimmed.range( + of: #"\s*\([^)]*\d[^)]*\)$"#, + options: .regularExpression) + { + return String(trimmed[.. String { + let compact = self.compactVersion(raw) + if compact.isEmpty { return compact } + if compact.lowercased().hasPrefix("v") { return compact } + if let first = compact.unicodeScalars.first, CharacterSet.decimalDigits.contains(first) { + return "v\(compact)" + } + return compact + } + + private static func versionLabels(_ entry: NodeInfo, compact: Bool = true) -> [String] { + let (core, ui) = self.resolveVersions(entry) + var labels: [String] = [] + if let core { + let label = compact ? self.compactVersion(core) : self.shortVersionLabel(core) + labels.append("core \(label)") + } + if let ui { + let label = compact ? self.compactVersion(ui) : self.shortVersionLabel(ui) + labels.append("ui \(label)") + } + return labels + } + + private static func resolveVersions(_ entry: NodeInfo) -> (core: String?, ui: String?) { + let core = entry.coreVersion?.nonEmpty + let ui = entry.uiVersion?.nonEmpty + if core != nil || ui != nil { + return (core, ui) + } + guard let legacy = entry.version?.nonEmpty else { return (nil, nil) } + if self.isHeadlessPlatform(entry) { + return (legacy, nil) + } + return (nil, legacy) + } + + private static func isHeadlessPlatform(_ entry: NodeInfo) -> Bool { + let raw = entry.platform?.trimmingCharacters(in: .whitespacesAndNewlines).lowercased() ?? "" + if raw == "darwin" || raw == "linux" || raw == "win32" || raw == "windows" { return true } + return false + } + + static func leadingSymbol(_ entry: NodeInfo) -> String { + if self.isGateway(entry) { + return self.safeSystemSymbol( + "antenna.radiowaves.left.and.right", + fallback: "dot.radiowaves.left.and.right") + } + if let family = entry.deviceFamily?.lowercased() { + if family.contains("mac") { + return self.safeSystemSymbol("laptopcomputer", fallback: "laptopcomputer") + } + if family.contains("iphone") { return self.safeSystemSymbol("iphone", fallback: "iphone") } + if family.contains("ipad") { return self.safeSystemSymbol("ipad", fallback: "ipad") } + } + if let platform = entry.platform?.lowercased() { + if platform.contains("mac") { return self.safeSystemSymbol("laptopcomputer", fallback: "laptopcomputer") } + if platform.contains("ios") { return self.safeSystemSymbol("iphone", fallback: "iphone") } + if platform.contains("android") { return self.safeSystemSymbol("cpu", fallback: "cpu") } + } + return "cpu" + } + + static func isAndroid(_ entry: NodeInfo) -> Bool { + let family = entry.deviceFamily?.trimmingCharacters(in: .whitespacesAndNewlines).lowercased() + if family == "android" { return true } + let platform = entry.platform?.trimmingCharacters(in: .whitespacesAndNewlines).lowercased() + return platform?.contains("android") == true + } + + private static func safeSystemSymbol(_ preferred: String, fallback: String) -> String { + if NSImage(systemSymbolName: preferred, accessibilityDescription: nil) != nil { return preferred } + return fallback + } +} + +struct NodeMenuRowView: View { + let entry: NodeInfo + let width: CGFloat + @Environment(\.menuItemHighlighted) private var isHighlighted + + private var primaryColor: Color { + self.isHighlighted ? Color(nsColor: .selectedMenuItemTextColor) : .primary + } + + private var secondaryColor: Color { + self.isHighlighted ? Color(nsColor: .selectedMenuItemTextColor).opacity(0.85) : .secondary + } + + var body: some View { + HStack(alignment: .center, spacing: 10) { + self.leadingIcon + .frame(width: 22, height: 22, alignment: .center) + + VStack(alignment: .leading, spacing: 2) { + HStack(alignment: .firstTextBaseline, spacing: 8) { + Text(NodeMenuEntryFormatter.primaryName(self.entry)) + .font(.callout.weight(NodeMenuEntryFormatter.isConnected(self.entry) ? .semibold : .regular)) + .foregroundStyle(self.primaryColor) + .lineLimit(1) + .truncationMode(.middle) + .layoutPriority(1) + + Spacer(minLength: 8) + + HStack(alignment: .firstTextBaseline, spacing: 6) { + if let right = NodeMenuEntryFormatter.headlineRight(self.entry) { + Text(right) + .font(.caption.monospacedDigit()) + .foregroundStyle(self.secondaryColor) + .lineLimit(1) + .truncationMode(.middle) + .layoutPriority(2) + } + + Image(systemName: "chevron.right") + .font(.caption.weight(.semibold)) + .foregroundStyle(self.secondaryColor) + .padding(.leading, 2) + } + } + + HStack(alignment: .firstTextBaseline, spacing: 8) { + Text(NodeMenuEntryFormatter.detailLeft(self.entry)) + .font(.caption) + .foregroundStyle(self.secondaryColor) + .lineLimit(1) + .truncationMode(.middle) + + Spacer(minLength: 0) + + if let version = NodeMenuEntryFormatter.detailRightVersion(self.entry) { + Text(version) + .font(.caption.monospacedDigit()) + .foregroundStyle(self.secondaryColor) + .lineLimit(1) + .truncationMode(.middle) + } + } + .frame(maxWidth: .infinity, alignment: .leading) + } + .frame(maxWidth: .infinity, alignment: .leading) + } + .padding(.vertical, 8) + .padding(.leading, 18) + .padding(.trailing, 12) + .frame(width: max(1, self.width), alignment: .leading) + } + + @ViewBuilder + private var leadingIcon: some View { + if NodeMenuEntryFormatter.isAndroid(self.entry) { + AndroidMark() + .foregroundStyle(self.secondaryColor) + } else { + Image(systemName: NodeMenuEntryFormatter.leadingSymbol(self.entry)) + .font(.system(size: 18, weight: .regular)) + .foregroundStyle(self.secondaryColor) + } + } +} + +struct AndroidMark: View { + var body: some View { + GeometryReader { geo in + let w = geo.size.width + let h = geo.size.height + let headHeight = h * 0.68 + let headWidth = w * 0.92 + let headX = (w - headWidth) * 0.5 + let headY = (h - headHeight) * 0.5 + let corner = min(w, h) * 0.18 + RoundedRectangle(cornerRadius: corner, style: .continuous) + .frame(width: headWidth, height: headHeight) + .position(x: headX + headWidth * 0.5, y: headY + headHeight * 0.5) + } + } +} + +struct NodeMenuMultilineView: View { + let label: String + let value: String + let width: CGFloat + @Environment(\.menuItemHighlighted) private var isHighlighted + + private var primaryColor: Color { + self.isHighlighted ? Color(nsColor: .selectedMenuItemTextColor) : .primary + } + + private var secondaryColor: Color { + self.isHighlighted ? Color(nsColor: .selectedMenuItemTextColor).opacity(0.85) : .secondary + } + + var body: some View { + VStack(alignment: .leading, spacing: 4) { + Text("\(self.label):") + .font(.caption.weight(.semibold)) + .foregroundStyle(self.secondaryColor) + + Text(self.value) + .font(.caption) + .foregroundStyle(self.primaryColor) + .multilineTextAlignment(.leading) + .fixedSize(horizontal: false, vertical: true) + } + .padding(.vertical, 6) + .padding(.leading, 18) + .padding(.trailing, 12) + .frame(width: max(1, self.width), alignment: .leading) + } +} diff --git a/apps/macos/Sources/OpenClaw/NodesStore.swift b/apps/macos/Sources/OpenClaw/NodesStore.swift new file mode 100644 index 0000000000000000000000000000000000000000..6ea5fbe90876d1b5f7c28e4fc328e40356ebb5f2 --- /dev/null +++ b/apps/macos/Sources/OpenClaw/NodesStore.swift @@ -0,0 +1,102 @@ +import Foundation +import Observation +import OSLog + +struct NodeInfo: Identifiable, Codable { + let nodeId: String + let displayName: String? + let platform: String? + let version: String? + let coreVersion: String? + let uiVersion: String? + let deviceFamily: String? + let modelIdentifier: String? + let remoteIp: String? + let caps: [String]? + let commands: [String]? + let permissions: [String: Bool]? + let paired: Bool? + let connected: Bool? + + var id: String { self.nodeId } + var isConnected: Bool { self.connected ?? false } + var isPaired: Bool { self.paired ?? false } +} + +private struct NodeListResponse: Codable { + let ts: Double? + let nodes: [NodeInfo] +} + +@MainActor +@Observable +final class NodesStore { + static let shared = NodesStore() + + var nodes: [NodeInfo] = [] + var lastError: String? + var statusMessage: String? + var isLoading = false + + private let logger = Logger(subsystem: "ai.openclaw", category: "nodes") + private var task: Task? + private let interval: TimeInterval = 30 + private var startCount = 0 + + func start() { + self.startCount += 1 + guard self.startCount == 1 else { return } + guard self.task == nil else { return } + self.task = Task.detached { [weak self] in + guard let self else { return } + await self.refresh() + while !Task.isCancelled { + try? await Task.sleep(nanoseconds: UInt64(self.interval * 1_000_000_000)) + await self.refresh() + } + } + } + + func stop() { + guard self.startCount > 0 else { return } + self.startCount -= 1 + guard self.startCount == 0 else { return } + self.task?.cancel() + self.task = nil + } + + func refresh() async { + if self.isLoading { return } + self.statusMessage = nil + self.isLoading = true + defer { self.isLoading = false } + do { + let data = try await GatewayConnection.shared.requestRaw(method: "node.list", params: nil, timeoutMs: 8000) + let decoded = try JSONDecoder().decode(NodeListResponse.self, from: data) + self.nodes = decoded.nodes + self.lastError = nil + self.statusMessage = nil + } catch { + if Self.isCancelled(error) { + self.logger.debug("node.list cancelled; keeping last nodes") + if self.nodes.isEmpty { + self.statusMessage = "Refreshing devices…" + } + self.lastError = nil + return + } + self.logger.error("node.list failed \(error.localizedDescription, privacy: .public)") + self.nodes = [] + self.lastError = error.localizedDescription + self.statusMessage = nil + } + } + + private static func isCancelled(_ error: Error) -> Bool { + if error is CancellationError { return true } + if let urlError = error as? URLError, urlError.code == .cancelled { return true } + let nsError = error as NSError + if nsError.domain == NSURLErrorDomain, nsError.code == NSURLErrorCancelled { return true } + return false + } +} diff --git a/apps/macos/Sources/OpenClaw/NotificationManager.swift b/apps/macos/Sources/OpenClaw/NotificationManager.swift new file mode 100644 index 0000000000000000000000000000000000000000..f522e6317643ab2253f9cf95a6ceaa5e6aeea6e4 --- /dev/null +++ b/apps/macos/Sources/OpenClaw/NotificationManager.swift @@ -0,0 +1,66 @@ +import OpenClawIPC +import Foundation +import Security +import UserNotifications + +@MainActor +struct NotificationManager { + private let logger = Logger(subsystem: "ai.openclaw", category: "notifications") + + private static let hasTimeSensitiveEntitlement: Bool = { + guard let task = SecTaskCreateFromSelf(nil) else { return false } + let key = "com.apple.developer.usernotifications.time-sensitive" as CFString + guard let val = SecTaskCopyValueForEntitlement(task, key, nil) else { return false } + return (val as? Bool) == true + }() + + func send(title: String, body: String, sound: String?, priority: NotificationPriority? = nil) async -> Bool { + let center = UNUserNotificationCenter.current() + let status = await center.notificationSettings() + if status.authorizationStatus == .notDetermined { + let granted = try? await center.requestAuthorization(options: [.alert, .sound, .badge]) + if granted != true { + self.logger.warning("notification permission denied (request)") + return false + } + } else if status.authorizationStatus != .authorized { + self.logger.warning("notification permission denied status=\(status.authorizationStatus.rawValue)") + return false + } + + let content = UNMutableNotificationContent() + content.title = title + content.body = body + if let soundName = sound, !soundName.isEmpty { + content.sound = UNNotificationSound(named: UNNotificationSoundName(soundName)) + } + + // Set interruption level based on priority + if let priority { + switch priority { + case .passive: + content.interruptionLevel = .passive + case .active: + content.interruptionLevel = .active + case .timeSensitive: + if Self.hasTimeSensitiveEntitlement { + content.interruptionLevel = .timeSensitive + } else { + self.logger.debug( + "time-sensitive notification requested without entitlement; falling back to active") + content.interruptionLevel = .active + } + } + } + + let req = UNNotificationRequest(identifier: UUID().uuidString, content: content, trigger: nil) + do { + try await center.add(req) + self.logger.debug("notification queued") + return true + } catch { + self.logger.error("notification send failed: \(error.localizedDescription)") + return false + } + } +} diff --git a/apps/macos/Sources/OpenClaw/NotifyOverlay.swift b/apps/macos/Sources/OpenClaw/NotifyOverlay.swift new file mode 100644 index 0000000000000000000000000000000000000000..1191c7e22227a71380cf720d3919c38a33325bad --- /dev/null +++ b/apps/macos/Sources/OpenClaw/NotifyOverlay.swift @@ -0,0 +1,190 @@ +import AppKit +import Observation +import QuartzCore +import SwiftUI + +/// Lightweight, borderless panel for in-app "toast" notifications (bypasses macOS Notification Center). +@MainActor +@Observable +final class NotifyOverlayController { + static let shared = NotifyOverlayController() + + private(set) var model = Model() + var isVisible: Bool { self.model.isVisible } + + struct Model { + var title: String = "" + var body: String = "" + var isVisible: Bool = false + } + + private var window: NSPanel? + private var hostingView: NSHostingView? + private var dismissTask: Task? + + private let width: CGFloat = 360 + private let padding: CGFloat = 12 + private let maxHeight: CGFloat = 220 + private let minHeight: CGFloat = 64 + + func present(title: String, body: String, autoDismissAfter: TimeInterval = 6) { + self.dismissTask?.cancel() + self.model.title = title + self.model.body = body + self.ensureWindow() + self.hostingView?.rootView = NotifyOverlayView(controller: self) + self.presentWindow() + + if autoDismissAfter > 0 { + self.dismissTask = Task { [weak self] in + try? await Task.sleep(nanoseconds: UInt64(autoDismissAfter * 1_000_000_000)) + await MainActor.run { self?.dismiss() } + } + } + } + + func dismiss() { + self.dismissTask?.cancel() + self.dismissTask = nil + guard let window else { return } + + let target = window.frame.offsetBy(dx: 8, dy: 6) + NSAnimationContext.runAnimationGroup { context in + context.duration = 0.16 + context.timingFunction = CAMediaTimingFunction(name: .easeOut) + window.animator().setFrame(target, display: true) + window.animator().alphaValue = 0 + } completionHandler: { + Task { @MainActor in + window.orderOut(nil) + self.model.isVisible = false + } + } + } + + // MARK: - Private + + private func presentWindow() { + self.ensureWindow() + self.hostingView?.rootView = NotifyOverlayView(controller: self) + let target = self.targetFrame() + + guard let window else { return } + if !self.model.isVisible { + self.model.isVisible = true + let start = target.offsetBy(dx: 0, dy: -6) + window.setFrame(start, display: true) + window.alphaValue = 0 + window.orderFrontRegardless() + NSAnimationContext.runAnimationGroup { context in + context.duration = 0.18 + context.timingFunction = CAMediaTimingFunction(name: .easeOut) + window.animator().setFrame(target, display: true) + window.animator().alphaValue = 1 + } + } else { + self.updateWindowFrame(animate: true) + window.orderFrontRegardless() + } + } + + private func ensureWindow() { + if self.window != nil { return } + let panel = NSPanel( + contentRect: NSRect(x: 0, y: 0, width: self.width, height: self.minHeight), + styleMask: [.nonactivatingPanel, .borderless], + backing: .buffered, + defer: false) + panel.isOpaque = false + panel.backgroundColor = .clear + panel.hasShadow = true + panel.level = .statusBar + panel.collectionBehavior = [.canJoinAllSpaces, .fullScreenAuxiliary, .transient] + panel.hidesOnDeactivate = false + panel.isMovable = false + panel.isFloatingPanel = true + panel.becomesKeyOnlyIfNeeded = true + panel.titleVisibility = .hidden + panel.titlebarAppearsTransparent = true + + let host = NSHostingView(rootView: NotifyOverlayView(controller: self)) + host.translatesAutoresizingMaskIntoConstraints = false + panel.contentView = host + self.hostingView = host + self.window = panel + } + + private func targetFrame() -> NSRect { + guard let screen = NSScreen.main else { return .zero } + let height = self.measuredHeight() + let size = NSSize(width: self.width, height: height) + let visible = screen.visibleFrame + let origin = CGPoint(x: visible.maxX - size.width - 8, y: visible.maxY - size.height - 8) + return NSRect(origin: origin, size: size) + } + + private func updateWindowFrame(animate: Bool = false) { + guard let window else { return } + let frame = self.targetFrame() + if animate { + NSAnimationContext.runAnimationGroup { context in + context.duration = 0.12 + context.timingFunction = CAMediaTimingFunction(name: .easeOut) + window.animator().setFrame(frame, display: true) + } + } else { + window.setFrame(frame, display: true) + } + } + + private func measuredHeight() -> CGFloat { + let maxWidth = self.width - self.padding * 2 + let titleFont = NSFont.systemFont(ofSize: 13, weight: .semibold) + let bodyFont = NSFont.systemFont(ofSize: 12, weight: .regular) + + let titleRect = (self.model.title as NSString).boundingRect( + with: CGSize(width: maxWidth, height: .greatestFiniteMagnitude), + options: [.usesLineFragmentOrigin, .usesFontLeading], + attributes: [.font: titleFont], + context: nil) + + let bodyRect = (self.model.body as NSString).boundingRect( + with: CGSize(width: maxWidth, height: .greatestFiniteMagnitude), + options: [.usesLineFragmentOrigin, .usesFontLeading], + attributes: [.font: bodyFont], + context: nil) + + let contentHeight = ceil(titleRect.height + 6 + bodyRect.height) + let total = contentHeight + self.padding * 2 + return max(self.minHeight, min(total, self.maxHeight)) + } +} + +private struct NotifyOverlayView: View { + var controller: NotifyOverlayController + + var body: some View { + VStack(alignment: .leading, spacing: 6) { + Text(self.controller.model.title) + .font(.system(size: 13, weight: .semibold)) + .foregroundStyle(.primary) + .lineLimit(1) + + Text(self.controller.model.body) + .font(.system(size: 12)) + .foregroundStyle(.secondary) + .lineLimit(4) + .fixedSize(horizontal: false, vertical: true) + } + .padding(12) + .background( + RoundedRectangle(cornerRadius: 12, style: .continuous) + .fill(.regularMaterial)) + .overlay( + RoundedRectangle(cornerRadius: 12, style: .continuous) + .strokeBorder(Color.black.opacity(0.08), lineWidth: 1)) + .onTapGesture { + self.controller.dismiss() + } + } +} diff --git a/apps/macos/Sources/OpenClaw/Onboarding.swift b/apps/macos/Sources/OpenClaw/Onboarding.swift new file mode 100644 index 0000000000000000000000000000000000000000..def8af4b2197d411d04d1476bfe457956d4b08b3 --- /dev/null +++ b/apps/macos/Sources/OpenClaw/Onboarding.swift @@ -0,0 +1,184 @@ +import AppKit +import OpenClawChatUI +import OpenClawDiscovery +import OpenClawIPC +import Combine +import Observation +import SwiftUI + +enum UIStrings { + static let welcomeTitle = "Welcome to OpenClaw" +} + +@MainActor +final class OnboardingController { + static let shared = OnboardingController() + private var window: NSWindow? + + func show() { + if ProcessInfo.processInfo.isNixMode { + // Nix mode is fully declarative; onboarding would suggest interactive setup that doesn't apply. + UserDefaults.standard.set(true, forKey: "openclaw.onboardingSeen") + UserDefaults.standard.set(currentOnboardingVersion, forKey: onboardingVersionKey) + AppStateStore.shared.onboardingSeen = true + return + } + if let window { + DockIconManager.shared.temporarilyShowDock() + window.makeKeyAndOrderFront(nil) + NSApp.activate(ignoringOtherApps: true) + return + } + let hosting = NSHostingController(rootView: OnboardingView()) + let window = NSWindow(contentViewController: hosting) + window.title = UIStrings.welcomeTitle + window.setContentSize(NSSize(width: OnboardingView.windowWidth, height: OnboardingView.windowHeight)) + window.styleMask = [.titled, .closable, .fullSizeContentView] + window.titlebarAppearsTransparent = true + window.titleVisibility = .hidden + window.isMovableByWindowBackground = true + window.center() + DockIconManager.shared.temporarilyShowDock() + window.makeKeyAndOrderFront(nil) + NSApp.activate(ignoringOtherApps: true) + self.window = window + } + + func close() { + self.window?.close() + self.window = nil + } + + func restart() { + self.close() + self.show() + } +} + +struct OnboardingView: View { + @Environment(\.openSettings) var openSettings + @State var currentPage = 0 + @State var isRequesting = false + @State var installingCLI = false + @State var cliStatus: String? + @State var copied = false + @State var monitoringPermissions = false + @State var monitoringDiscovery = false + @State var cliInstalled = false + @State var cliInstallLocation: String? + @State var workspacePath: String = "" + @State var workspaceStatus: String? + @State var workspaceApplying = false + @State var anthropicAuthPKCE: AnthropicOAuth.PKCE? + @State var anthropicAuthCode: String = "" + @State var anthropicAuthStatus: String? + @State var anthropicAuthBusy = false + @State var anthropicAuthConnected = false + @State var anthropicAuthVerifying = false + @State var anthropicAuthVerified = false + @State var anthropicAuthVerificationAttempted = false + @State var anthropicAuthVerificationFailed = false + @State var anthropicAuthVerifiedAt: Date? + @State var anthropicAuthDetectedStatus: OpenClawOAuthStore.AnthropicOAuthStatus = .missingFile + @State var anthropicAuthAutoDetectClipboard = true + @State var anthropicAuthAutoConnectClipboard = true + @State var anthropicAuthLastPasteboardChangeCount = NSPasteboard.general.changeCount + @State var monitoringAuth = false + @State var authMonitorTask: Task? + @State var needsBootstrap = false + @State var didAutoKickoff = false + @State var showAdvancedConnection = false + @State var preferredGatewayID: String? + @State var gatewayDiscovery: GatewayDiscoveryModel + @State var onboardingChatModel: OpenClawChatViewModel + @State var onboardingSkillsModel = SkillsSettingsModel() + @State var onboardingWizard = OnboardingWizardModel() + @State var didLoadOnboardingSkills = false + @State var localGatewayProbe: LocalGatewayProbe? + @Bindable var state: AppState + var permissionMonitor: PermissionMonitor + + static let windowWidth: CGFloat = 630 + static let windowHeight: CGFloat = 752 // ~+10% to fit full onboarding content + + let pageWidth: CGFloat = Self.windowWidth + let contentHeight: CGFloat = 460 + let connectionPageIndex = 1 + let anthropicAuthPageIndex = 2 + let wizardPageIndex = 3 + let onboardingChatPageIndex = 8 + + static let clipboardPoll: AnyPublisher = { + if ProcessInfo.processInfo.isRunningTests { + return Empty(completeImmediately: false).eraseToAnyPublisher() + } + return Timer.publish(every: 0.4, on: .main, in: .common) + .autoconnect() + .eraseToAnyPublisher() + }() + + let permissionsPageIndex = 5 + static func pageOrder( + for mode: AppState.ConnectionMode, + showOnboardingChat: Bool) -> [Int] + { + switch mode { + case .remote: + // Remote setup doesn't need local gateway/CLI/workspace setup pages, + // and WhatsApp/Telegram setup is optional. + showOnboardingChat ? [0, 1, 5, 8, 9] : [0, 1, 5, 9] + case .unconfigured: + showOnboardingChat ? [0, 1, 8, 9] : [0, 1, 9] + case .local: + showOnboardingChat ? [0, 1, 3, 5, 8, 9] : [0, 1, 3, 5, 9] + } + } + + var showOnboardingChat: Bool { + self.state.connectionMode == .local && self.needsBootstrap + } + + var pageOrder: [Int] { + Self.pageOrder(for: self.state.connectionMode, showOnboardingChat: self.showOnboardingChat) + } + + var pageCount: Int { self.pageOrder.count } + var activePageIndex: Int { + self.activePageIndex(for: self.currentPage) + } + + var buttonTitle: String { self.currentPage == self.pageCount - 1 ? "Finish" : "Next" } + var wizardPageOrderIndex: Int? { self.pageOrder.firstIndex(of: self.wizardPageIndex) } + var isWizardBlocking: Bool { + self.activePageIndex == self.wizardPageIndex && !self.onboardingWizard.isComplete + } + + var canAdvance: Bool { !self.isWizardBlocking } + var devLinkCommand: String { + let version = GatewayEnvironment.expectedGatewayVersionString() ?? "latest" + return "npm install -g openclaw@\(version)" + } + + struct LocalGatewayProbe: Equatable { + let port: Int + let pid: Int32 + let command: String + let expected: Bool + } + + init( + state: AppState = AppStateStore.shared, + permissionMonitor: PermissionMonitor = .shared, + discoveryModel: GatewayDiscoveryModel = GatewayDiscoveryModel( + localDisplayName: InstanceIdentity.displayName, + filterLocalGateways: false)) + { + self.state = state + self.permissionMonitor = permissionMonitor + self._gatewayDiscovery = State(initialValue: discoveryModel) + self._onboardingChatModel = State( + initialValue: OpenClawChatViewModel( + sessionKey: "onboarding", + transport: MacGatewayChatTransport())) + } +} diff --git a/apps/macos/Sources/OpenClaw/OnboardingView+Actions.swift b/apps/macos/Sources/OpenClaw/OnboardingView+Actions.swift new file mode 100644 index 0000000000000000000000000000000000000000..bfffc39f15e722011bc2fcf5c9d27f305a213396 --- /dev/null +++ b/apps/macos/Sources/OpenClaw/OnboardingView+Actions.swift @@ -0,0 +1,148 @@ +import AppKit +import OpenClawDiscovery +import OpenClawIPC +import Foundation +import SwiftUI + +extension OnboardingView { + func selectLocalGateway() { + self.state.connectionMode = .local + self.preferredGatewayID = nil + self.showAdvancedConnection = false + GatewayDiscoveryPreferences.setPreferredStableID(nil) + } + + func selectUnconfiguredGateway() { + Task { await self.onboardingWizard.cancelIfRunning() } + self.state.connectionMode = .unconfigured + self.preferredGatewayID = nil + self.showAdvancedConnection = false + GatewayDiscoveryPreferences.setPreferredStableID(nil) + } + + func selectRemoteGateway(_ gateway: GatewayDiscoveryModel.DiscoveredGateway) { + Task { await self.onboardingWizard.cancelIfRunning() } + self.preferredGatewayID = gateway.stableID + GatewayDiscoveryPreferences.setPreferredStableID(gateway.stableID) + + if self.state.remoteTransport == .direct { + if let url = GatewayDiscoveryHelpers.directUrl(for: gateway) { + self.state.remoteUrl = url + } + } else if let host = GatewayDiscoveryHelpers.sanitizedTailnetHost(gateway.tailnetDns) ?? gateway.lanHost { + let user = NSUserName() + self.state.remoteTarget = GatewayDiscoveryModel.buildSSHTarget( + user: user, + host: host, + port: gateway.sshPort) + OpenClawConfigFile.setRemoteGatewayUrl(host: host, port: gateway.gatewayPort) + } + self.state.remoteCliPath = gateway.cliPath ?? "" + + self.state.connectionMode = .remote + MacNodeModeCoordinator.shared.setPreferredGatewayStableID(gateway.stableID) + } + + func openSettings(tab: SettingsTab) { + SettingsTabRouter.request(tab) + self.openSettings() + DispatchQueue.main.async { + NotificationCenter.default.post(name: .openclawSelectSettingsTab, object: tab) + } + } + + func handleBack() { + withAnimation { + self.currentPage = max(0, self.currentPage - 1) + } + } + + func handleNext() { + if self.isWizardBlocking { return } + if self.currentPage < self.pageCount - 1 { + withAnimation { self.currentPage += 1 } + } else { + self.finish() + } + } + + func finish() { + UserDefaults.standard.set(true, forKey: "openclaw.onboardingSeen") + UserDefaults.standard.set(currentOnboardingVersion, forKey: onboardingVersionKey) + OnboardingController.shared.close() + } + + func copyToPasteboard(_ text: String) { + let pb = NSPasteboard.general + pb.clearContents() + pb.setString(text, forType: .string) + self.copied = true + DispatchQueue.main.asyncAfter(deadline: .now() + 1.2) { self.copied = false } + } + + func startAnthropicOAuth() { + guard !self.anthropicAuthBusy else { return } + self.anthropicAuthBusy = true + defer { self.anthropicAuthBusy = false } + + do { + let pkce = try AnthropicOAuth.generatePKCE() + self.anthropicAuthPKCE = pkce + let url = AnthropicOAuth.buildAuthorizeURL(pkce: pkce) + NSWorkspace.shared.open(url) + self.anthropicAuthStatus = "Browser opened. After approving, paste the `code#state` value here." + } catch { + self.anthropicAuthStatus = "Failed to start OAuth: \(error.localizedDescription)" + } + } + + @MainActor + func finishAnthropicOAuth() async { + guard !self.anthropicAuthBusy else { return } + guard let pkce = self.anthropicAuthPKCE else { return } + self.anthropicAuthBusy = true + defer { self.anthropicAuthBusy = false } + + guard let parsed = AnthropicOAuthCodeState.parse(from: self.anthropicAuthCode) else { + self.anthropicAuthStatus = "OAuth failed: missing or invalid code/state." + return + } + + do { + let creds = try await AnthropicOAuth.exchangeCode( + code: parsed.code, + state: parsed.state, + verifier: pkce.verifier) + try OpenClawOAuthStore.saveAnthropicOAuth(creds) + self.refreshAnthropicOAuthStatus() + self.anthropicAuthStatus = "Connected. OpenClaw can now use Claude." + } catch { + self.anthropicAuthStatus = "OAuth failed: \(error.localizedDescription)" + } + } + + func pollAnthropicClipboardIfNeeded() { + guard self.currentPage == self.anthropicAuthPageIndex else { return } + guard self.anthropicAuthPKCE != nil else { return } + guard !self.anthropicAuthBusy else { return } + guard self.anthropicAuthAutoDetectClipboard else { return } + + let pb = NSPasteboard.general + let changeCount = pb.changeCount + guard changeCount != self.anthropicAuthLastPasteboardChangeCount else { return } + self.anthropicAuthLastPasteboardChangeCount = changeCount + + guard let raw = pb.string(forType: .string), !raw.isEmpty else { return } + guard let parsed = AnthropicOAuthCodeState.parse(from: raw) else { return } + guard let pkce = self.anthropicAuthPKCE, parsed.state == pkce.verifier else { return } + + let next = "\(parsed.code)#\(parsed.state)" + if self.anthropicAuthCode != next { + self.anthropicAuthCode = next + self.anthropicAuthStatus = "Detected `code#state` from clipboard." + } + + guard self.anthropicAuthAutoConnectClipboard else { return } + Task { await self.finishAnthropicOAuth() } + } +} diff --git a/apps/macos/Sources/OpenClaw/OnboardingView+Chat.swift b/apps/macos/Sources/OpenClaw/OnboardingView+Chat.swift new file mode 100644 index 0000000000000000000000000000000000000000..f95da4ffbb5dd2bc12954e571b7e3c29b62b5568 --- /dev/null +++ b/apps/macos/Sources/OpenClaw/OnboardingView+Chat.swift @@ -0,0 +1,26 @@ +import Foundation + +extension OnboardingView { + func maybeKickoffOnboardingChat(for pageIndex: Int) { + guard pageIndex == self.onboardingChatPageIndex else { return } + guard self.showOnboardingChat else { return } + guard !self.didAutoKickoff else { return } + self.didAutoKickoff = true + + Task { @MainActor in + for _ in 0..<20 { + if !self.onboardingChatModel.isLoading { break } + try? await Task.sleep(nanoseconds: 200_000_000) + } + guard self.onboardingChatModel.messages.isEmpty else { return } + let kickoff = + "Hi! I just installed OpenClaw and you’re my brand‑new agent. " + + "Please start the first‑run ritual from BOOTSTRAP.md, ask one question at a time, " + + "and before we talk about WhatsApp/Telegram, visit soul.md with me to craft SOUL.md: " + + "ask what matters to me and how you should be. Then guide me through choosing " + + "how we should talk (web‑only, WhatsApp, or Telegram)." + self.onboardingChatModel.input = kickoff + self.onboardingChatModel.send() + } + } +} diff --git a/apps/macos/Sources/OpenClaw/OnboardingView+Layout.swift b/apps/macos/Sources/OpenClaw/OnboardingView+Layout.swift new file mode 100644 index 0000000000000000000000000000000000000000..ce87e211ce4412be6e254ba60c519089241e61a4 --- /dev/null +++ b/apps/macos/Sources/OpenClaw/OnboardingView+Layout.swift @@ -0,0 +1,234 @@ +import AppKit +import SwiftUI + +extension OnboardingView { + var body: some View { + VStack(spacing: 0) { + GlowingOpenClawIcon(size: 130, glowIntensity: 0.28) + .offset(y: 10) + .frame(height: 145) + + GeometryReader { _ in + HStack(spacing: 0) { + ForEach(self.pageOrder, id: \.self) { pageIndex in + self.pageView(for: pageIndex) + .frame(width: self.pageWidth) + } + } + .offset(x: CGFloat(-self.currentPage) * self.pageWidth) + .animation( + .interactiveSpring(response: 0.5, dampingFraction: 0.86, blendDuration: 0.25), + value: self.currentPage) + .frame(height: self.contentHeight, alignment: .top) + .clipped() + } + .frame(height: self.contentHeight) + + Spacer(minLength: 0) + self.navigationBar + } + .frame(width: self.pageWidth, height: Self.windowHeight) + .background(Color(NSColor.windowBackgroundColor)) + .onAppear { + self.currentPage = 0 + self.updateMonitoring(for: 0) + } + .onChange(of: self.currentPage) { _, newValue in + self.updateMonitoring(for: self.activePageIndex(for: newValue)) + } + .onChange(of: self.state.connectionMode) { _, _ in + let oldActive = self.activePageIndex + self.reconcilePageForModeChange(previousActivePageIndex: oldActive) + self.updateDiscoveryMonitoring(for: self.activePageIndex) + } + .onChange(of: self.needsBootstrap) { _, _ in + if self.currentPage >= self.pageOrder.count { + self.currentPage = max(0, self.pageOrder.count - 1) + } + } + .onChange(of: self.onboardingWizard.isComplete) { _, newValue in + guard newValue, self.activePageIndex == self.wizardPageIndex else { return } + self.handleNext() + } + .onDisappear { + self.stopPermissionMonitoring() + self.stopDiscovery() + self.stopAuthMonitoring() + Task { await self.onboardingWizard.cancelIfRunning() } + } + .task { + await self.refreshPerms() + self.refreshCLIStatus() + await self.loadWorkspaceDefaults() + await self.ensureDefaultWorkspace() + self.refreshAnthropicOAuthStatus() + self.refreshBootstrapStatus() + self.preferredGatewayID = GatewayDiscoveryPreferences.preferredStableID() + } + } + + func activePageIndex(for pageCursor: Int) -> Int { + guard !self.pageOrder.isEmpty else { return 0 } + let clamped = min(max(0, pageCursor), self.pageOrder.count - 1) + return self.pageOrder[clamped] + } + + func reconcilePageForModeChange(previousActivePageIndex: Int) { + if let exact = self.pageOrder.firstIndex(of: previousActivePageIndex) { + withAnimation { self.currentPage = exact } + return + } + if let next = self.pageOrder.firstIndex(where: { $0 > previousActivePageIndex }) { + withAnimation { self.currentPage = next } + return + } + withAnimation { self.currentPage = max(0, self.pageOrder.count - 1) } + } + + var navigationBar: some View { + let wizardLockIndex = self.wizardPageOrderIndex + return HStack(spacing: 20) { + ZStack(alignment: .leading) { + Button(action: {}, label: { + Label("Back", systemImage: "chevron.left").labelStyle(.iconOnly) + }) + .buttonStyle(.plain) + .opacity(0) + .disabled(true) + + if self.currentPage > 0 { + Button(action: self.handleBack, label: { + Label("Back", systemImage: "chevron.left") + .labelStyle(.iconOnly) + }) + .buttonStyle(.plain) + .foregroundColor(.secondary) + .opacity(0.8) + .transition(.opacity.combined(with: .scale(scale: 0.9))) + } + } + .frame(minWidth: 80, alignment: .leading) + + Spacer() + + HStack(spacing: 8) { + ForEach(0.. (wizardLockIndex ?? 0) + Button { + withAnimation { self.currentPage = index } + } label: { + Circle() + .fill(index == self.currentPage ? Color.accentColor : Color.gray.opacity(0.3)) + .frame(width: 8, height: 8) + } + .buttonStyle(.plain) + .disabled(isLocked) + .opacity(isLocked ? 0.3 : 1) + } + } + + Spacer() + + Button(action: self.handleNext) { + Text(self.buttonTitle) + .frame(minWidth: 88) + } + .keyboardShortcut(.return) + .buttonStyle(.borderedProminent) + .disabled(!self.canAdvance) + } + .padding(.horizontal, 28) + .padding(.bottom, 13) + .frame(minHeight: 60, alignment: .bottom) + } + + func onboardingPage(@ViewBuilder _ content: () -> some View) -> some View { + let scrollIndicatorGutter: CGFloat = 18 + return ScrollView { + VStack(spacing: 16) { + content() + Spacer(minLength: 0) + } + .frame(maxWidth: .infinity, alignment: .top) + .padding(.trailing, scrollIndicatorGutter) + } + .scrollIndicators(.automatic) + .padding(.horizontal, 28) + .frame(width: self.pageWidth, alignment: .top) + } + + func onboardingCard( + spacing: CGFloat = 12, + padding: CGFloat = 16, + @ViewBuilder _ content: () -> some View) -> some View + { + VStack(alignment: .leading, spacing: spacing) { + content() + } + .padding(padding) + .frame(maxWidth: .infinity, alignment: .leading) + .background( + RoundedRectangle(cornerRadius: 16, style: .continuous) + .fill(Color(NSColor.controlBackgroundColor)) + .shadow(color: .black.opacity(0.06), radius: 8, y: 3)) + } + + func onboardingGlassCard( + spacing: CGFloat = 12, + padding: CGFloat = 16, + @ViewBuilder _ content: () -> some View) -> some View + { + let shape = RoundedRectangle(cornerRadius: 16, style: .continuous) + return VStack(alignment: .leading, spacing: spacing) { + content() + } + .padding(padding) + .frame(maxWidth: .infinity, alignment: .leading) + .background(Color.clear) + .clipShape(shape) + .overlay(shape.strokeBorder(Color.white.opacity(0.10), lineWidth: 1)) + } + + func featureRow(title: String, subtitle: String, systemImage: String) -> some View { + HStack(alignment: .top, spacing: 12) { + Image(systemName: systemImage) + .font(.title3.weight(.semibold)) + .foregroundStyle(Color.accentColor) + .frame(width: 26) + VStack(alignment: .leading, spacing: 4) { + Text(title).font(.headline) + Text(subtitle) + .font(.subheadline) + .foregroundStyle(.secondary) + } + } + .padding(.vertical, 4) + } + + func featureActionRow( + title: String, + subtitle: String, + systemImage: String, + buttonTitle: String, + action: @escaping () -> Void) -> some View + { + HStack(alignment: .top, spacing: 12) { + Image(systemName: systemImage) + .font(.title3.weight(.semibold)) + .foregroundStyle(Color.accentColor) + .frame(width: 26) + VStack(alignment: .leading, spacing: 4) { + Text(title).font(.headline) + Text(subtitle) + .font(.subheadline) + .foregroundStyle(.secondary) + Button(buttonTitle, action: action) + .buttonStyle(.link) + .padding(.top, 2) + } + Spacer(minLength: 0) + } + .padding(.vertical, 4) + } +} diff --git a/apps/macos/Sources/OpenClaw/OnboardingView+Monitoring.swift b/apps/macos/Sources/OpenClaw/OnboardingView+Monitoring.swift new file mode 100644 index 0000000000000000000000000000000000000000..64ddc332e4ac0d9cc3c5b40491c6dbc56910b73a --- /dev/null +++ b/apps/macos/Sources/OpenClaw/OnboardingView+Monitoring.swift @@ -0,0 +1,178 @@ +import OpenClawIPC +import Foundation + +extension OnboardingView { + @MainActor + func refreshPerms() async { + await self.permissionMonitor.refreshNow() + } + + @MainActor + func request(_ cap: Capability) async { + guard !self.isRequesting else { return } + self.isRequesting = true + defer { isRequesting = false } + _ = await PermissionManager.ensure([cap], interactive: true) + await self.refreshPerms() + } + + func updatePermissionMonitoring(for pageIndex: Int) { + let shouldMonitor = pageIndex == self.permissionsPageIndex + if shouldMonitor, !self.monitoringPermissions { + self.monitoringPermissions = true + PermissionMonitor.shared.register() + } else if !shouldMonitor, self.monitoringPermissions { + self.monitoringPermissions = false + PermissionMonitor.shared.unregister() + } + } + + func updateDiscoveryMonitoring(for pageIndex: Int) { + let isConnectionPage = pageIndex == self.connectionPageIndex + let shouldMonitor = isConnectionPage + if shouldMonitor, !self.monitoringDiscovery { + self.monitoringDiscovery = true + Task { @MainActor in + try? await Task.sleep(nanoseconds: 150_000_000) + guard self.monitoringDiscovery else { return } + self.gatewayDiscovery.start() + await self.refreshLocalGatewayProbe() + } + } else if !shouldMonitor, self.monitoringDiscovery { + self.monitoringDiscovery = false + self.gatewayDiscovery.stop() + } + } + + func updateMonitoring(for pageIndex: Int) { + self.updatePermissionMonitoring(for: pageIndex) + self.updateDiscoveryMonitoring(for: pageIndex) + self.updateAuthMonitoring(for: pageIndex) + self.maybeKickoffOnboardingChat(for: pageIndex) + } + + func stopPermissionMonitoring() { + guard self.monitoringPermissions else { return } + self.monitoringPermissions = false + PermissionMonitor.shared.unregister() + } + + func stopDiscovery() { + guard self.monitoringDiscovery else { return } + self.monitoringDiscovery = false + self.gatewayDiscovery.stop() + } + + func updateAuthMonitoring(for pageIndex: Int) { + let shouldMonitor = pageIndex == self.anthropicAuthPageIndex && self.state.connectionMode == .local + if shouldMonitor, !self.monitoringAuth { + self.monitoringAuth = true + self.startAuthMonitoring() + } else if !shouldMonitor, self.monitoringAuth { + self.stopAuthMonitoring() + } + } + + func startAuthMonitoring() { + self.refreshAnthropicOAuthStatus() + self.authMonitorTask?.cancel() + self.authMonitorTask = Task { + while !Task.isCancelled { + await MainActor.run { self.refreshAnthropicOAuthStatus() } + try? await Task.sleep(nanoseconds: 1_000_000_000) + } + } + } + + func stopAuthMonitoring() { + self.monitoringAuth = false + self.authMonitorTask?.cancel() + self.authMonitorTask = nil + } + + func installCLI() async { + guard !self.installingCLI else { return } + self.installingCLI = true + defer { installingCLI = false } + await CLIInstaller.install { message in + self.cliStatus = message + } + self.refreshCLIStatus() + } + + func refreshCLIStatus() { + let installLocation = CLIInstaller.installedLocation() + self.cliInstallLocation = installLocation + self.cliInstalled = installLocation != nil + } + + func refreshLocalGatewayProbe() async { + let port = GatewayEnvironment.gatewayPort() + let desc = await PortGuardian.shared.describe(port: port) + await MainActor.run { + guard let desc else { + self.localGatewayProbe = nil + return + } + let command = desc.command.trimmingCharacters(in: .whitespacesAndNewlines) + let expectedTokens = ["node", "openclaw", "tsx", "pnpm", "bun"] + let lower = command.lowercased() + let expected = expectedTokens.contains { lower.contains($0) } + self.localGatewayProbe = LocalGatewayProbe( + port: port, + pid: desc.pid, + command: command, + expected: expected) + } + } + + func refreshAnthropicOAuthStatus() { + _ = OpenClawOAuthStore.importLegacyAnthropicOAuthIfNeeded() + let previous = self.anthropicAuthDetectedStatus + let status = OpenClawOAuthStore.anthropicOAuthStatus() + self.anthropicAuthDetectedStatus = status + self.anthropicAuthConnected = status.isConnected + + if previous != status { + self.anthropicAuthVerified = false + self.anthropicAuthVerificationAttempted = false + self.anthropicAuthVerificationFailed = false + self.anthropicAuthVerifiedAt = nil + } + } + + @MainActor + func verifyAnthropicOAuthIfNeeded(force: Bool = false) async { + guard self.state.connectionMode == .local else { return } + guard self.anthropicAuthDetectedStatus.isConnected else { return } + if self.anthropicAuthVerified, !force { return } + if self.anthropicAuthVerifying { return } + if self.anthropicAuthVerificationAttempted, !force { return } + + self.anthropicAuthVerificationAttempted = true + self.anthropicAuthVerifying = true + self.anthropicAuthVerificationFailed = false + defer { self.anthropicAuthVerifying = false } + + guard let refresh = OpenClawOAuthStore.loadAnthropicOAuthRefreshToken(), !refresh.isEmpty else { + self.anthropicAuthStatus = "OAuth verification failed: missing refresh token." + self.anthropicAuthVerificationFailed = true + return + } + + do { + let updated = try await AnthropicOAuth.refresh(refreshToken: refresh) + try OpenClawOAuthStore.saveAnthropicOAuth(updated) + self.refreshAnthropicOAuthStatus() + self.anthropicAuthVerified = true + self.anthropicAuthVerifiedAt = Date() + self.anthropicAuthVerificationFailed = false + self.anthropicAuthStatus = "OAuth detected and verified." + } catch { + self.anthropicAuthVerified = false + self.anthropicAuthVerifiedAt = nil + self.anthropicAuthVerificationFailed = true + self.anthropicAuthStatus = "OAuth verification failed: \(error.localizedDescription)" + } + } +} diff --git a/apps/macos/Sources/OpenClaw/OnboardingView+Pages.swift b/apps/macos/Sources/OpenClaw/OnboardingView+Pages.swift new file mode 100644 index 0000000000000000000000000000000000000000..48a1baf7ec3a3dda9b407c998fa84772e6558650 --- /dev/null +++ b/apps/macos/Sources/OpenClaw/OnboardingView+Pages.swift @@ -0,0 +1,843 @@ +import AppKit +import OpenClawChatUI +import OpenClawDiscovery +import OpenClawIPC +import SwiftUI + +extension OnboardingView { + @ViewBuilder + func pageView(for pageIndex: Int) -> some View { + switch pageIndex { + case 0: + self.welcomePage() + case 1: + self.connectionPage() + case 2: + self.anthropicAuthPage() + case 3: + self.wizardPage() + case 5: + self.permissionsPage() + case 6: + self.cliPage() + case 8: + self.onboardingChatPage() + case 9: + self.readyPage() + default: + EmptyView() + } + } + + func welcomePage() -> some View { + self.onboardingPage { + VStack(spacing: 22) { + Text("Welcome to OpenClaw") + .font(.largeTitle.weight(.semibold)) + Text("OpenClaw is a powerful personal AI assistant that can connect to WhatsApp or Telegram.") + .font(.body) + .foregroundStyle(.secondary) + .multilineTextAlignment(.center) + .lineLimit(2) + .frame(maxWidth: 560) + .fixedSize(horizontal: false, vertical: true) + + self.onboardingCard(spacing: 10, padding: 14) { + HStack(alignment: .top, spacing: 12) { + Image(systemName: "exclamationmark.triangle.fill") + .font(.title3.weight(.semibold)) + .foregroundStyle(Color(nsColor: .systemOrange)) + .frame(width: 22) + .padding(.top, 1) + + VStack(alignment: .leading, spacing: 6) { + Text("Security notice") + .font(.headline) + Text( + "The connected AI agent (e.g. Claude) can trigger powerful actions on your Mac, " + + "including running commands, reading/writing files, and capturing screenshots — " + + "depending on the permissions you grant.\n\n" + + "Only enable OpenClaw if you understand the risks and trust the prompts and " + + "integrations you use.") + .font(.subheadline) + .foregroundStyle(.secondary) + .fixedSize(horizontal: false, vertical: true) + } + } + } + .frame(maxWidth: 520) + } + .padding(.top, 16) + } + } + + func connectionPage() -> some View { + self.onboardingPage { + Text("Choose your Gateway") + .font(.largeTitle.weight(.semibold)) + Text( + "OpenClaw uses a single Gateway that stays running. Pick this Mac, " + + "connect to a discovered gateway nearby, or configure later.") + .font(.body) + .foregroundStyle(.secondary) + .multilineTextAlignment(.center) + .lineLimit(2) + .frame(maxWidth: 520) + .fixedSize(horizontal: false, vertical: true) + + self.onboardingCard(spacing: 12, padding: 14) { + VStack(alignment: .leading, spacing: 10) { + let localSubtitle: String = { + guard let probe = self.localGatewayProbe else { + return "Gateway starts automatically on this Mac." + } + let base = probe.expected + ? "Existing gateway detected" + : "Port \(probe.port) already in use" + let command = probe.command.isEmpty ? "" : " (\(probe.command) pid \(probe.pid))" + return "\(base)\(command). Will attach." + }() + self.connectionChoiceButton( + title: "This Mac", + subtitle: localSubtitle, + selected: self.state.connectionMode == .local) + { + self.selectLocalGateway() + } + + Divider().padding(.vertical, 4) + + HStack(spacing: 8) { + Image(systemName: "dot.radiowaves.left.and.right") + .font(.caption) + .foregroundStyle(.secondary) + Text(self.gatewayDiscovery.statusText) + .font(.caption) + .foregroundStyle(.secondary) + if self.gatewayDiscovery.gateways.isEmpty { + ProgressView().controlSize(.small) + Button("Refresh") { + self.gatewayDiscovery.refreshWideAreaFallbackNow(timeoutSeconds: 5.0) + } + .buttonStyle(.link) + .help("Retry Tailscale discovery (DNS-SD).") + } + Spacer(minLength: 0) + } + + if self.gatewayDiscovery.gateways.isEmpty { + Text("Searching for nearby gateways…") + .font(.caption) + .foregroundStyle(.secondary) + .padding(.leading, 4) + } else { + VStack(alignment: .leading, spacing: 6) { + Text("Nearby gateways") + .font(.caption) + .foregroundStyle(.secondary) + .padding(.leading, 4) + ForEach(self.gatewayDiscovery.gateways.prefix(6)) { gateway in + self.connectionChoiceButton( + title: gateway.displayName, + subtitle: self.gatewaySubtitle(for: gateway), + selected: self.isSelectedGateway(gateway)) + { + self.selectRemoteGateway(gateway) + } + } + } + .padding(8) + .background( + RoundedRectangle(cornerRadius: 10, style: .continuous) + .fill(Color(NSColor.controlBackgroundColor))) + } + + self.connectionChoiceButton( + title: "Configure later", + subtitle: "Don’t start the Gateway yet.", + selected: self.state.connectionMode == .unconfigured) + { + self.selectUnconfiguredGateway() + } + + Button(self.showAdvancedConnection ? "Hide Advanced" : "Advanced…") { + withAnimation(.spring(response: 0.25, dampingFraction: 0.9)) { + self.showAdvancedConnection.toggle() + } + if self.showAdvancedConnection, self.state.connectionMode != .remote { + self.state.connectionMode = .remote + } + } + .buttonStyle(.link) + + if self.showAdvancedConnection { + let labelWidth: CGFloat = 110 + let fieldWidth: CGFloat = 320 + + VStack(alignment: .leading, spacing: 10) { + Grid(alignment: .leading, horizontalSpacing: 12, verticalSpacing: 8) { + GridRow { + Text("Transport") + .font(.callout.weight(.semibold)) + .frame(width: labelWidth, alignment: .leading) + Picker("Transport", selection: self.$state.remoteTransport) { + Text("SSH tunnel").tag(AppState.RemoteTransport.ssh) + Text("Direct (ws/wss)").tag(AppState.RemoteTransport.direct) + } + .pickerStyle(.segmented) + .frame(width: fieldWidth) + } + if self.state.remoteTransport == .direct { + GridRow { + Text("Gateway URL") + .font(.callout.weight(.semibold)) + .frame(width: labelWidth, alignment: .leading) + TextField("wss://gateway.example.ts.net", text: self.$state.remoteUrl) + .textFieldStyle(.roundedBorder) + .frame(width: fieldWidth) + } + } + if self.state.remoteTransport == .ssh { + GridRow { + Text("SSH target") + .font(.callout.weight(.semibold)) + .frame(width: labelWidth, alignment: .leading) + TextField("user@host[:port]", text: self.$state.remoteTarget) + .textFieldStyle(.roundedBorder) + .frame(width: fieldWidth) + } + if let message = CommandResolver.sshTargetValidationMessage(self.state.remoteTarget) { + GridRow { + Text("") + .frame(width: labelWidth, alignment: .leading) + Text(message) + .font(.caption) + .foregroundStyle(.red) + .frame(width: fieldWidth, alignment: .leading) + } + } + GridRow { + Text("Identity file") + .font(.callout.weight(.semibold)) + .frame(width: labelWidth, alignment: .leading) + TextField("/Users/you/.ssh/id_ed25519", text: self.$state.remoteIdentity) + .textFieldStyle(.roundedBorder) + .frame(width: fieldWidth) + } + GridRow { + Text("Project root") + .font(.callout.weight(.semibold)) + .frame(width: labelWidth, alignment: .leading) + TextField("/home/you/Projects/openclaw", text: self.$state.remoteProjectRoot) + .textFieldStyle(.roundedBorder) + .frame(width: fieldWidth) + } + GridRow { + Text("CLI path") + .font(.callout.weight(.semibold)) + .frame(width: labelWidth, alignment: .leading) + TextField( + "/Applications/OpenClaw.app/.../openclaw", + text: self.$state.remoteCliPath) + .textFieldStyle(.roundedBorder) + .frame(width: fieldWidth) + } + } + } + + Text(self.state.remoteTransport == .direct + ? "Tip: use Tailscale Serve so the gateway has a valid HTTPS cert." + : "Tip: keep Tailscale enabled so your gateway stays reachable.") + .font(.footnote) + .foregroundStyle(.secondary) + .lineLimit(1) + } + .transition(.opacity.combined(with: .move(edge: .top))) + } + } + } + } + } + + func gatewaySubtitle(for gateway: GatewayDiscoveryModel.DiscoveredGateway) -> String? { + if self.state.remoteTransport == .direct { + return GatewayDiscoveryHelpers.directUrl(for: gateway) ?? "Gateway pairing only" + } + if let host = GatewayDiscoveryHelpers.sanitizedTailnetHost(gateway.tailnetDns) ?? gateway.lanHost { + let portSuffix = gateway.sshPort != 22 ? " · ssh \(gateway.sshPort)" : "" + return "\(host)\(portSuffix)" + } + return "Gateway pairing only" + } + + func isSelectedGateway(_ gateway: GatewayDiscoveryModel.DiscoveredGateway) -> Bool { + guard self.state.connectionMode == .remote else { return false } + let preferred = self.preferredGatewayID ?? GatewayDiscoveryPreferences.preferredStableID() + return preferred == gateway.stableID + } + + func connectionChoiceButton( + title: String, + subtitle: String?, + selected: Bool, + action: @escaping () -> Void) -> some View + { + Button { + withAnimation(.spring(response: 0.25, dampingFraction: 0.9)) { + action() + } + } label: { + HStack(alignment: .center, spacing: 10) { + VStack(alignment: .leading, spacing: 2) { + Text(title) + .font(.callout.weight(.semibold)) + .lineLimit(1) + .truncationMode(.tail) + if let subtitle { + Text(subtitle) + .font(.caption.monospaced()) + .foregroundStyle(.secondary) + .lineLimit(1) + .truncationMode(.middle) + } + } + Spacer(minLength: 0) + if selected { + Image(systemName: "checkmark.circle.fill") + .foregroundStyle(Color.accentColor) + } else { + Image(systemName: "arrow.right.circle") + .foregroundStyle(.secondary) + } + } + .padding(.horizontal, 10) + .padding(.vertical, 8) + .frame(maxWidth: .infinity, alignment: .leading) + .background( + RoundedRectangle(cornerRadius: 10, style: .continuous) + .fill(selected ? Color.accentColor.opacity(0.12) : Color.clear)) + .overlay( + RoundedRectangle(cornerRadius: 10, style: .continuous) + .strokeBorder( + selected ? Color.accentColor.opacity(0.45) : Color.clear, + lineWidth: 1)) + } + .buttonStyle(.plain) + } + + func anthropicAuthPage() -> some View { + self.onboardingPage { + Text("Connect Claude") + .font(.largeTitle.weight(.semibold)) + Text("Give your model the token it needs!") + .font(.body) + .foregroundStyle(.secondary) + .multilineTextAlignment(.center) + .frame(maxWidth: 540) + .fixedSize(horizontal: false, vertical: true) + Text("OpenClaw supports any model — we strongly recommend Opus 4.5 for the best experience.") + .font(.callout) + .foregroundStyle(.secondary) + .multilineTextAlignment(.center) + .frame(maxWidth: 540) + .fixedSize(horizontal: false, vertical: true) + + self.onboardingCard(spacing: 12, padding: 16) { + HStack(alignment: .center, spacing: 10) { + Circle() + .fill(self.anthropicAuthVerified ? Color.green : Color.orange) + .frame(width: 10, height: 10) + Text( + self.anthropicAuthConnected + ? (self.anthropicAuthVerified + ? "Claude connected (OAuth) — verified" + : "Claude connected (OAuth)") + : "Not connected yet") + .font(.headline) + Spacer() + } + + if self.anthropicAuthConnected, self.anthropicAuthVerifying { + Text("Verifying OAuth…") + .font(.caption) + .foregroundStyle(.secondary) + .fixedSize(horizontal: false, vertical: true) + } else if !self.anthropicAuthConnected { + Text(self.anthropicAuthDetectedStatus.shortDescription) + .font(.caption) + .foregroundStyle(.secondary) + .fixedSize(horizontal: false, vertical: true) + } else if self.anthropicAuthVerified, let date = self.anthropicAuthVerifiedAt { + Text("Detected working OAuth (\(date.formatted(date: .abbreviated, time: .shortened))).") + .font(.caption) + .foregroundStyle(.secondary) + .fixedSize(horizontal: false, vertical: true) + } + + Text( + "This lets OpenClaw use Claude immediately. Credentials are stored at " + + "`~/.openclaw/credentials/oauth.json` (owner-only).") + .font(.subheadline) + .foregroundStyle(.secondary) + .fixedSize(horizontal: false, vertical: true) + + HStack(spacing: 12) { + Text(OpenClawOAuthStore.oauthURL().path) + .font(.caption) + .foregroundStyle(.secondary) + .lineLimit(1) + .truncationMode(.middle) + + Spacer() + + Button("Reveal") { + NSWorkspace.shared.activateFileViewerSelecting([OpenClawOAuthStore.oauthURL()]) + } + .buttonStyle(.bordered) + + Button("Refresh") { + self.refreshAnthropicOAuthStatus() + } + .buttonStyle(.bordered) + } + + Divider().padding(.vertical, 2) + + HStack(spacing: 12) { + if !self.anthropicAuthVerified { + if self.anthropicAuthConnected { + Button("Verify") { + Task { await self.verifyAnthropicOAuthIfNeeded(force: true) } + } + .buttonStyle(.borderedProminent) + .disabled(self.anthropicAuthBusy || self.anthropicAuthVerifying) + + if self.anthropicAuthVerificationFailed { + Button("Re-auth (OAuth)") { + self.startAnthropicOAuth() + } + .buttonStyle(.bordered) + .disabled(self.anthropicAuthBusy || self.anthropicAuthVerifying) + } + } else { + Button { + self.startAnthropicOAuth() + } label: { + if self.anthropicAuthBusy { + ProgressView() + } else { + Text("Open Claude sign-in (OAuth)") + } + } + .buttonStyle(.borderedProminent) + .disabled(self.anthropicAuthBusy) + } + } + } + + if !self.anthropicAuthVerified, self.anthropicAuthPKCE != nil { + VStack(alignment: .leading, spacing: 8) { + Text("Paste the `code#state` value") + .font(.headline) + TextField("code#state", text: self.$anthropicAuthCode) + .textFieldStyle(.roundedBorder) + + Toggle("Auto-detect from clipboard", isOn: self.$anthropicAuthAutoDetectClipboard) + .font(.caption) + .foregroundStyle(.secondary) + .disabled(self.anthropicAuthBusy) + + Toggle("Auto-connect when detected", isOn: self.$anthropicAuthAutoConnectClipboard) + .font(.caption) + .foregroundStyle(.secondary) + .disabled(self.anthropicAuthBusy) + + Button("Connect") { + Task { await self.finishAnthropicOAuth() } + } + .buttonStyle(.bordered) + .disabled( + self.anthropicAuthBusy || + self.anthropicAuthCode.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty) + } + .onReceive(Self.clipboardPoll) { _ in + self.pollAnthropicClipboardIfNeeded() + } + } + + self.onboardingCard(spacing: 8, padding: 12) { + Text("API key (advanced)") + .font(.headline) + Text( + "You can also use an Anthropic API key, but this UI is instructions-only for now " + + "(GUI apps don’t automatically inherit your shell env vars like `ANTHROPIC_API_KEY`).") + .font(.subheadline) + .foregroundStyle(.secondary) + .fixedSize(horizontal: false, vertical: true) + } + .shadow(color: .clear, radius: 0) + .background(Color.clear) + + if let status = self.anthropicAuthStatus { + Text(status) + .font(.caption) + .foregroundStyle(.secondary) + .fixedSize(horizontal: false, vertical: true) + } + } + } + .task { await self.verifyAnthropicOAuthIfNeeded() } + } + + func permissionsPage() -> some View { + self.onboardingPage { + Text("Grant permissions") + .font(.largeTitle.weight(.semibold)) + Text("These macOS permissions let OpenClaw automate apps and capture context on this Mac.") + .font(.body) + .foregroundStyle(.secondary) + .multilineTextAlignment(.center) + .frame(maxWidth: 520) + .fixedSize(horizontal: false, vertical: true) + + self.onboardingCard(spacing: 8, padding: 12) { + ForEach(Capability.allCases, id: \.self) { cap in + PermissionRow( + capability: cap, + status: self.permissionMonitor.status[cap] ?? false, + compact: true) + { + Task { await self.request(cap) } + } + } + + HStack(spacing: 12) { + Button { + Task { await self.refreshPerms() } + } label: { + Label("Refresh", systemImage: "arrow.clockwise") + } + .buttonStyle(.bordered) + .controlSize(.small) + .help("Refresh status") + if self.isRequesting { + ProgressView() + .controlSize(.small) + } + } + .padding(.top, 4) + } + } + } + + func cliPage() -> some View { + self.onboardingPage { + Text("Install the CLI") + .font(.largeTitle.weight(.semibold)) + Text("Required for local mode: installs `openclaw` so launchd can run the gateway.") + .font(.body) + .foregroundStyle(.secondary) + .multilineTextAlignment(.center) + .frame(maxWidth: 520) + .fixedSize(horizontal: false, vertical: true) + + self.onboardingCard(spacing: 10) { + HStack(spacing: 12) { + Button { + Task { await self.installCLI() } + } label: { + let title = self.cliInstalled ? "Reinstall CLI" : "Install CLI" + ZStack { + Text(title) + .opacity(self.installingCLI ? 0 : 1) + if self.installingCLI { + ProgressView() + .controlSize(.mini) + } + } + .frame(minWidth: 120) + } + .buttonStyle(.borderedProminent) + .disabled(self.installingCLI) + + Button(self.copied ? "Copied" : "Copy install command") { + self.copyToPasteboard(self.devLinkCommand) + } + .disabled(self.installingCLI) + + if self.cliInstalled, let loc = self.cliInstallLocation { + Label("Installed at \(loc)", systemImage: "checkmark.circle.fill") + .font(.footnote) + .foregroundStyle(.green) + } + } + + if let cliStatus { + Text(cliStatus) + .font(.caption) + .foregroundStyle(.secondary) + } else if !self.cliInstalled, self.cliInstallLocation == nil { + Text( + """ + Installs a user-space Node 22+ runtime and the CLI (no Homebrew). + Rerun anytime to reinstall or update. + """) + .font(.footnote) + .foregroundStyle(.secondary) + } + } + } + } + + func workspacePage() -> some View { + self.onboardingPage { + Text("Agent workspace") + .font(.largeTitle.weight(.semibold)) + Text( + "OpenClaw runs the agent from a dedicated workspace so it can load `AGENTS.md` " + + "and write files there without mixing into your other projects.") + .font(.body) + .foregroundStyle(.secondary) + .multilineTextAlignment(.center) + .frame(maxWidth: 560) + .fixedSize(horizontal: false, vertical: true) + + self.onboardingCard(spacing: 10) { + if self.state.connectionMode == .remote { + Text("Remote gateway detected") + .font(.headline) + Text( + "Create the workspace on the remote host (SSH in first). " + + "The macOS app can’t write files on your gateway over SSH yet.") + .font(.subheadline) + .foregroundStyle(.secondary) + + Button(self.copied ? "Copied" : "Copy setup command") { + self.copyToPasteboard(self.workspaceBootstrapCommand) + } + .buttonStyle(.bordered) + } else { + VStack(alignment: .leading, spacing: 8) { + Text("Workspace folder") + .font(.headline) + TextField( + AgentWorkspace.displayPath(for: OpenClawConfigFile.defaultWorkspaceURL()), + text: self.$workspacePath) + .textFieldStyle(.roundedBorder) + + HStack(spacing: 12) { + Button { + Task { await self.applyWorkspace() } + } label: { + if self.workspaceApplying { + ProgressView() + } else { + Text("Create workspace") + } + } + .buttonStyle(.borderedProminent) + .disabled(self.workspaceApplying) + + Button("Open folder") { + let url = AgentWorkspace.resolveWorkspaceURL(from: self.workspacePath) + NSWorkspace.shared.open(url) + } + .buttonStyle(.bordered) + .disabled(self.workspaceApplying) + + Button("Save in config") { + Task { + let url = AgentWorkspace.resolveWorkspaceURL(from: self.workspacePath) + let saved = await self.saveAgentWorkspace(AgentWorkspace.displayPath(for: url)) + if saved { + self.workspaceStatus = + "Saved to ~/.openclaw/openclaw.json (agents.defaults.workspace)" + } + } + } + .buttonStyle(.bordered) + .disabled(self.workspaceApplying) + } + } + + if let workspaceStatus { + Text(workspaceStatus) + .font(.caption) + .foregroundStyle(.secondary) + .lineLimit(2) + } else { + Text( + "Tip: edit AGENTS.md in this folder to shape the assistant’s behavior. " + + "For backup, make the workspace a private git repo so your agent’s " + + "“memory” is versioned.") + .font(.caption) + .foregroundStyle(.secondary) + .lineLimit(2) + } + } + } + } + } + + func onboardingChatPage() -> some View { + VStack(spacing: 16) { + Text("Meet your agent") + .font(.largeTitle.weight(.semibold)) + Text( + "This is a dedicated onboarding chat. Your agent will introduce itself, " + + "learn who you are, and help you connect WhatsApp or Telegram if you want.") + .font(.body) + .foregroundStyle(.secondary) + .multilineTextAlignment(.center) + .frame(maxWidth: 520) + .fixedSize(horizontal: false, vertical: true) + + self.onboardingGlassCard(padding: 8) { + OpenClawChatView(viewModel: self.onboardingChatModel, style: .onboarding) + .frame(maxHeight: .infinity) + } + .frame(maxHeight: .infinity) + } + .padding(.horizontal, 28) + .frame(width: self.pageWidth, height: self.contentHeight, alignment: .top) + } + + func readyPage() -> some View { + self.onboardingPage { + Text("All set") + .font(.largeTitle.weight(.semibold)) + self.onboardingCard { + if self.state.connectionMode == .unconfigured { + self.featureRow( + title: "Configure later", + subtitle: "Pick Local or Remote in Settings → General whenever you’re ready.", + systemImage: "gearshape") + Divider() + .padding(.vertical, 6) + } + if self.state.connectionMode == .remote { + self.featureRow( + title: "Remote gateway checklist", + subtitle: """ + On your gateway host: install/update the `openclaw` package and make sure credentials exist + (typically `~/.openclaw/credentials/oauth.json`). Then connect again if needed. + """, + systemImage: "network") + Divider() + .padding(.vertical, 6) + } + self.featureRow( + title: "Open the menu bar panel", + subtitle: "Click the OpenClaw menu bar icon for quick chat and status.", + systemImage: "bubble.left.and.bubble.right") + self.featureActionRow( + title: "Connect WhatsApp or Telegram", + subtitle: "Open Settings → Channels to link channels and monitor status.", + systemImage: "link", + buttonTitle: "Open Settings → Channels") + { + self.openSettings(tab: .channels) + } + self.featureRow( + title: "Try Voice Wake", + subtitle: "Enable Voice Wake in Settings for hands-free commands with a live transcript overlay.", + systemImage: "waveform.circle") + self.featureRow( + title: "Use the panel + Canvas", + subtitle: "Open the menu bar panel for quick chat; the agent can show previews " + + "and richer visuals in Canvas.", + systemImage: "rectangle.inset.filled.and.person.filled") + self.featureActionRow( + title: "Give your agent more powers", + subtitle: "Enable optional skills (Peekaboo, oracle, camsnap, …) from Settings → Skills.", + systemImage: "sparkles", + buttonTitle: "Open Settings → Skills") + { + self.openSettings(tab: .skills) + } + self.skillsOverview + Toggle("Launch at login", isOn: self.$state.launchAtLogin) + .onChange(of: self.state.launchAtLogin) { _, newValue in + AppStateStore.updateLaunchAtLogin(enabled: newValue) + } + } + } + .task { await self.maybeLoadOnboardingSkills() } + } + + private func maybeLoadOnboardingSkills() async { + guard !self.didLoadOnboardingSkills else { return } + self.didLoadOnboardingSkills = true + await self.onboardingSkillsModel.refresh() + } + + private var skillsOverview: some View { + VStack(alignment: .leading, spacing: 8) { + Divider() + .padding(.vertical, 6) + + HStack(spacing: 10) { + Text("Skills included") + .font(.headline) + Spacer(minLength: 0) + if self.onboardingSkillsModel.isLoading { + ProgressView() + .controlSize(.small) + } else { + Button("Refresh") { + Task { await self.onboardingSkillsModel.refresh() } + } + .buttonStyle(.link) + } + } + + if let error = self.onboardingSkillsModel.error { + VStack(alignment: .leading, spacing: 4) { + Text("Couldn’t load skills from the Gateway.") + .font(.footnote.weight(.semibold)) + .foregroundStyle(.orange) + Text( + "Make sure the Gateway is running and connected, " + + "then hit Refresh (or open Settings → Skills).") + .font(.footnote) + .foregroundStyle(.secondary) + .fixedSize(horizontal: false, vertical: true) + Text("Details: \(error)") + .font(.caption.monospaced()) + .foregroundStyle(.secondary) + .fixedSize(horizontal: false, vertical: true) + } + } else if self.onboardingSkillsModel.skills.isEmpty { + Text("No skills reported yet.") + .font(.footnote) + .foregroundStyle(.secondary) + } else { + ScrollView { + LazyVStack(alignment: .leading, spacing: 10) { + ForEach(self.onboardingSkillsModel.skills) { skill in + HStack(alignment: .top, spacing: 10) { + Text(skill.emoji ?? "✨") + .font(.callout) + .frame(width: 22, alignment: .leading) + VStack(alignment: .leading, spacing: 2) { + Text(skill.name) + .font(.callout.weight(.semibold)) + Text(skill.description) + .font(.footnote) + .foregroundStyle(.secondary) + .fixedSize(horizontal: false, vertical: true) + } + Spacer(minLength: 0) + } + } + } + .padding(10) + .background( + RoundedRectangle(cornerRadius: 12, style: .continuous) + .fill(Color(NSColor.windowBackgroundColor))) + } + .frame(maxHeight: 160) + } + } + } +} diff --git a/apps/macos/Sources/OpenClaw/OnboardingView+Testing.swift b/apps/macos/Sources/OpenClaw/OnboardingView+Testing.swift new file mode 100644 index 0000000000000000000000000000000000000000..cf8c3d0c78f67ff56320d77dff005bbdfbe34fbf --- /dev/null +++ b/apps/macos/Sources/OpenClaw/OnboardingView+Testing.swift @@ -0,0 +1,87 @@ +import OpenClawDiscovery +import SwiftUI + +#if DEBUG +@MainActor +extension OnboardingView { + static func exerciseForTesting() { + let state = AppState(preview: true) + let discovery = GatewayDiscoveryModel(localDisplayName: InstanceIdentity.displayName) + discovery.statusText = "Searching..." + let gateway = GatewayDiscoveryModel.DiscoveredGateway( + displayName: "Test Gateway", + lanHost: "gateway.local", + tailnetDns: "gateway.ts.net", + sshPort: 2222, + gatewayPort: 18789, + cliPath: "/usr/local/bin/openclaw", + stableID: "gateway-1", + debugID: "gateway-1", + isLocal: false) + discovery.gateways = [gateway] + + let view = OnboardingView( + state: state, + permissionMonitor: PermissionMonitor.shared, + discoveryModel: discovery) + view.needsBootstrap = true + view.localGatewayProbe = LocalGatewayProbe( + port: GatewayEnvironment.gatewayPort(), + pid: 123, + command: "openclaw-gateway", + expected: true) + view.showAdvancedConnection = true + view.preferredGatewayID = gateway.stableID + view.cliInstalled = true + view.cliInstallLocation = "/usr/local/bin/openclaw" + view.cliStatus = "Installed" + view.workspacePath = "/tmp/openclaw" + view.workspaceStatus = "Saved workspace" + view.anthropicAuthPKCE = AnthropicOAuth.PKCE(verifier: "verifier", challenge: "challenge") + view.anthropicAuthCode = "code#state" + view.anthropicAuthStatus = "Connected" + view.anthropicAuthDetectedStatus = .connected(expiresAtMs: 1_700_000_000_000) + view.anthropicAuthConnected = true + view.anthropicAuthAutoDetectClipboard = false + view.anthropicAuthAutoConnectClipboard = false + + view.state.connectionMode = .local + _ = view.welcomePage() + _ = view.connectionPage() + _ = view.anthropicAuthPage() + _ = view.wizardPage() + _ = view.permissionsPage() + _ = view.cliPage() + _ = view.workspacePage() + _ = view.onboardingChatPage() + _ = view.readyPage() + + view.selectLocalGateway() + view.selectRemoteGateway(gateway) + view.selectUnconfiguredGateway() + + view.state.connectionMode = .remote + _ = view.connectionPage() + _ = view.workspacePage() + + view.state.connectionMode = .unconfigured + _ = view.connectionPage() + + view.currentPage = 0 + view.handleNext() + view.handleBack() + + _ = view.onboardingPage { Text("Test") } + _ = view.onboardingCard { Text("Card") } + _ = view.featureRow(title: "Feature", subtitle: "Subtitle", systemImage: "sparkles") + _ = view.featureActionRow( + title: "Action", + subtitle: "Action subtitle", + systemImage: "gearshape", + buttonTitle: "Action", + action: {}) + _ = view.gatewaySubtitle(for: gateway) + _ = view.isSelectedGateway(gateway) + } +} +#endif diff --git a/apps/macos/Sources/OpenClaw/OnboardingView+Wizard.swift b/apps/macos/Sources/OpenClaw/OnboardingView+Wizard.swift new file mode 100644 index 0000000000000000000000000000000000000000..51424fdb78c853fb398c03c04fea0a1f7bbdf95c --- /dev/null +++ b/apps/macos/Sources/OpenClaw/OnboardingView+Wizard.swift @@ -0,0 +1,94 @@ +import OpenClawProtocol +import Observation +import SwiftUI + +extension OnboardingView { + func wizardPage() -> some View { + self.onboardingPage { + VStack(spacing: 16) { + Text("Setup Wizard") + .font(.largeTitle.weight(.semibold)) + Text("Follow the guided setup from the Gateway. This keeps onboarding in sync with the CLI.") + .font(.body) + .foregroundStyle(.secondary) + .multilineTextAlignment(.center) + .frame(maxWidth: 520) + + self.onboardingCard(spacing: 14, padding: 16) { + OnboardingWizardCardContent( + wizard: self.onboardingWizard, + mode: self.state.connectionMode, + workspacePath: self.workspacePath) + } + } + .task { + await self.onboardingWizard.startIfNeeded( + mode: self.state.connectionMode, + workspace: self.workspacePath.isEmpty ? nil : self.workspacePath) + } + } + } +} + +private struct OnboardingWizardCardContent: View { + @Bindable var wizard: OnboardingWizardModel + let mode: AppState.ConnectionMode + let workspacePath: String + + private enum CardState { + case error(String) + case starting + case step(WizardStep) + case complete + case waiting + } + + private var state: CardState { + if let error = wizard.errorMessage { return .error(error) } + if self.wizard.isStarting { return .starting } + if let step = wizard.currentStep { return .step(step) } + if self.wizard.isComplete { return .complete } + return .waiting + } + + var body: some View { + switch self.state { + case let .error(error): + Text("Wizard error") + .font(.headline) + Text(error) + .font(.subheadline) + .foregroundStyle(.secondary) + .fixedSize(horizontal: false, vertical: true) + Button("Retry") { + self.wizard.reset() + Task { + await self.wizard.startIfNeeded( + mode: self.mode, + workspace: self.workspacePath.isEmpty ? nil : self.workspacePath) + } + } + .buttonStyle(.borderedProminent) + case .starting: + HStack(spacing: 8) { + ProgressView() + Text("Starting wizard…") + .foregroundStyle(.secondary) + } + case let .step(step): + OnboardingWizardStepView( + step: step, + isSubmitting: self.wizard.isSubmitting) + { value in + Task { await self.wizard.submit(step: step, value: value) } + } + .id(step.id) + case .complete: + Text("Wizard complete. Continue to the next step.") + .font(.headline) + case .waiting: + Text("Waiting for wizard…") + .foregroundStyle(.secondary) + } + } +} diff --git a/apps/macos/Sources/OpenClaw/OnboardingView+Workspace.swift b/apps/macos/Sources/OpenClaw/OnboardingView+Workspace.swift new file mode 100644 index 0000000000000000000000000000000000000000..0b413433666b7acab2846cf663bfa4cf94ddea3e --- /dev/null +++ b/apps/macos/Sources/OpenClaw/OnboardingView+Workspace.swift @@ -0,0 +1,116 @@ +import Foundation + +extension OnboardingView { + func loadWorkspaceDefaults() async { + guard self.workspacePath.isEmpty else { return } + let configured = await self.loadAgentWorkspace() + let url = AgentWorkspace.resolveWorkspaceURL(from: configured) + self.workspacePath = AgentWorkspace.displayPath(for: url) + self.refreshBootstrapStatus() + } + + func ensureDefaultWorkspace() async { + guard self.state.connectionMode == .local else { return } + let configured = await self.loadAgentWorkspace() + let url = AgentWorkspace.resolveWorkspaceURL(from: configured) + switch AgentWorkspace.bootstrapSafety(for: url) { + case .safe: + do { + _ = try AgentWorkspace.bootstrap(workspaceURL: url) + if (configured ?? "").trimmingCharacters(in: .whitespacesAndNewlines).isEmpty { + await self.saveAgentWorkspace(AgentWorkspace.displayPath(for: url)) + } + } catch { + self.workspaceStatus = "Failed to create workspace: \(error.localizedDescription)" + } + case let .unsafe(reason): + self.workspaceStatus = "Workspace not touched: \(reason)" + } + self.refreshBootstrapStatus() + } + + func refreshBootstrapStatus() { + let url = AgentWorkspace.resolveWorkspaceURL(from: self.workspacePath) + self.needsBootstrap = AgentWorkspace.needsBootstrap(workspaceURL: url) + if self.needsBootstrap { + self.didAutoKickoff = false + } + } + + var workspaceBootstrapCommand: String { + let template = AgentWorkspace.defaultTemplate().trimmingCharacters(in: .whitespacesAndNewlines) + return """ + mkdir -p ~/.openclaw/workspace + cat > ~/.openclaw/workspace/AGENTS.md <<'EOF' + \(template) + EOF + """ + } + + func applyWorkspace() async { + guard !self.workspaceApplying else { return } + self.workspaceApplying = true + defer { self.workspaceApplying = false } + + do { + let url = AgentWorkspace.resolveWorkspaceURL(from: self.workspacePath) + if case let .unsafe(reason) = AgentWorkspace.bootstrapSafety(for: url) { + self.workspaceStatus = "Workspace not created: \(reason)" + return + } + _ = try AgentWorkspace.bootstrap(workspaceURL: url) + self.workspacePath = AgentWorkspace.displayPath(for: url) + self.workspaceStatus = "Workspace ready at \(self.workspacePath)" + self.refreshBootstrapStatus() + } catch { + self.workspaceStatus = "Failed to create workspace: \(error.localizedDescription)" + } + } + + private func loadAgentWorkspace() async -> String? { + let root = await ConfigStore.load() + let agents = root["agents"] as? [String: Any] + let defaults = agents?["defaults"] as? [String: Any] + return defaults?["workspace"] as? String + } + + @discardableResult + func saveAgentWorkspace(_ workspace: String?) async -> Bool { + let (success, errorMessage) = await OnboardingView.buildAndSaveWorkspace(workspace) + + if let errorMessage { + self.workspaceStatus = errorMessage + } + return success + } + + @MainActor + private static func buildAndSaveWorkspace(_ workspace: String?) async -> (Bool, String?) { + var root = await ConfigStore.load() + var agents = root["agents"] as? [String: Any] ?? [:] + var defaults = agents["defaults"] as? [String: Any] ?? [:] + let trimmed = workspace?.trimmingCharacters(in: .whitespacesAndNewlines) ?? "" + if trimmed.isEmpty { + defaults.removeValue(forKey: "workspace") + } else { + defaults["workspace"] = trimmed + } + if defaults.isEmpty { + agents.removeValue(forKey: "defaults") + } else { + agents["defaults"] = defaults + } + if agents.isEmpty { + root.removeValue(forKey: "agents") + } else { + root["agents"] = agents + } + do { + try await ConfigStore.save(root) + return (true, nil) + } catch { + let errorMessage = "Failed to save config: \(error.localizedDescription)" + return (false, errorMessage) + } + } +} diff --git a/apps/macos/Sources/OpenClaw/OnboardingWidgets.swift b/apps/macos/Sources/OpenClaw/OnboardingWidgets.swift new file mode 100644 index 0000000000000000000000000000000000000000..58d09ef66dc858e728690f1f88956fb7b47b550d --- /dev/null +++ b/apps/macos/Sources/OpenClaw/OnboardingWidgets.swift @@ -0,0 +1,65 @@ +import AppKit +import SwiftUI + +struct GlowingOpenClawIcon: View { + @Environment(\.scenePhase) private var scenePhase + + let size: CGFloat + let glowIntensity: Double + let enableFloating: Bool + + @State private var breathe = false + + init(size: CGFloat = 148, glowIntensity: Double = 0.35, enableFloating: Bool = true) { + self.size = size + self.glowIntensity = glowIntensity + self.enableFloating = enableFloating + } + + var body: some View { + let glowBlurRadius: CGFloat = 18 + let glowCanvasSize: CGFloat = self.size + 56 + ZStack { + Circle() + .fill( + LinearGradient( + colors: [ + Color.accentColor.opacity(self.glowIntensity), + Color.blue.opacity(self.glowIntensity * 0.6), + ], + startPoint: .topLeading, + endPoint: .bottomTrailing)) + .frame(width: glowCanvasSize, height: glowCanvasSize) + .padding(glowBlurRadius) + .blur(radius: glowBlurRadius) + .scaleEffect(self.breathe ? 1.08 : 0.96) + .opacity(0.84) + + Image(nsImage: NSApp.applicationIconImage) + .resizable() + .frame(width: self.size, height: self.size) + .clipShape(RoundedRectangle(cornerRadius: self.size * 0.22, style: .continuous)) + .shadow(color: .black.opacity(0.18), radius: 14, y: 6) + .scaleEffect(self.breathe ? 1.02 : 1.0) + } + .frame( + width: glowCanvasSize + (glowBlurRadius * 2), + height: glowCanvasSize + (glowBlurRadius * 2)) + .onAppear { self.updateBreatheAnimation() } + .onDisappear { self.breathe = false } + .onChange(of: self.scenePhase) { _, _ in + self.updateBreatheAnimation() + } + } + + private func updateBreatheAnimation() { + guard self.enableFloating, self.scenePhase == .active else { + self.breathe = false + return + } + guard !self.breathe else { return } + withAnimation(Animation.easeInOut(duration: 3.6).repeatForever(autoreverses: true)) { + self.breathe = true + } + } +} diff --git a/apps/macos/Sources/OpenClaw/OnboardingWizard.swift b/apps/macos/Sources/OpenClaw/OnboardingWizard.swift new file mode 100644 index 0000000000000000000000000000000000000000..412826650a66f651c3672b66674fa29aa3807deb --- /dev/null +++ b/apps/macos/Sources/OpenClaw/OnboardingWizard.swift @@ -0,0 +1,412 @@ +import OpenClawKit +import OpenClawProtocol +import Foundation +import Observation +import OSLog +import SwiftUI + +private let onboardingWizardLogger = Logger(subsystem: "ai.openclaw", category: "onboarding.wizard") + +// MARK: - Swift 6 AnyCodable Bridging Helpers + +// Bridge between OpenClawProtocol.AnyCodable and the local module to avoid +// Swift 6 strict concurrency type conflicts. + +private typealias ProtocolAnyCodable = OpenClawProtocol.AnyCodable + +private func bridgeToLocal(_ value: ProtocolAnyCodable) -> AnyCodable { + if let data = try? JSONEncoder().encode(value), + let decoded = try? JSONDecoder().decode(AnyCodable.self, from: data) + { + return decoded + } + return AnyCodable(value.value) +} + +private func bridgeToLocal(_ value: ProtocolAnyCodable?) -> AnyCodable? { + value.map(bridgeToLocal) +} + +@MainActor +@Observable +final class OnboardingWizardModel { + private(set) var sessionId: String? + private(set) var currentStep: WizardStep? + private(set) var status: String? + private(set) var errorMessage: String? + var isStarting = false + var isSubmitting = false + private var lastStartMode: AppState.ConnectionMode? + private var lastStartWorkspace: String? + private var restartAttempts = 0 + private let maxRestartAttempts = 1 + + var isComplete: Bool { self.status == "done" } + var isRunning: Bool { self.status == "running" } + + func reset() { + self.sessionId = nil + self.currentStep = nil + self.status = nil + self.errorMessage = nil + self.isStarting = false + self.isSubmitting = false + self.restartAttempts = 0 + self.lastStartMode = nil + self.lastStartWorkspace = nil + } + + func startIfNeeded(mode: AppState.ConnectionMode, workspace: String? = nil) async { + guard self.sessionId == nil, !self.isStarting else { return } + guard mode == .local else { return } + if self.shouldSkipWizard() { + self.sessionId = nil + self.currentStep = nil + self.status = "done" + self.errorMessage = nil + return + } + self.isStarting = true + self.errorMessage = nil + self.lastStartMode = mode + self.lastStartWorkspace = workspace + defer { self.isStarting = false } + + do { + GatewayProcessManager.shared.setActive(true) + if await GatewayProcessManager.shared.waitForGatewayReady(timeout: 12) == false { + throw NSError( + domain: "Gateway", + code: 1, + userInfo: [NSLocalizedDescriptionKey: "Gateway did not become ready. Check that it is running."]) + } + var params: [String: AnyCodable] = ["mode": AnyCodable("local")] + if let workspace, !workspace.isEmpty { + params["workspace"] = AnyCodable(workspace) + } + let res: WizardStartResult = try await GatewayConnection.shared.requestDecoded( + method: .wizardStart, + params: params) + self.applyStartResult(res) + } catch { + self.status = "error" + self.errorMessage = error.localizedDescription + onboardingWizardLogger.error("start failed: \(error.localizedDescription, privacy: .public)") + } + } + + func submit(step: WizardStep, value: AnyCodable?) async { + guard let sessionId, !self.isSubmitting else { return } + self.isSubmitting = true + self.errorMessage = nil + defer { self.isSubmitting = false } + + do { + var params: [String: AnyCodable] = ["sessionId": AnyCodable(sessionId)] + var answer: [String: AnyCodable] = ["stepId": AnyCodable(step.id)] + if let value { + answer["value"] = value + } + params["answer"] = AnyCodable(answer) + let res: WizardNextResult = try await GatewayConnection.shared.requestDecoded( + method: .wizardNext, + params: params) + self.applyNextResult(res) + } catch { + if self.restartIfSessionLost(error: error) { + return + } + self.status = "error" + self.errorMessage = error.localizedDescription + onboardingWizardLogger.error("submit failed: \(error.localizedDescription, privacy: .public)") + } + } + + func cancelIfRunning() async { + guard let sessionId, self.isRunning else { return } + do { + let res: WizardStatusResult = try await GatewayConnection.shared.requestDecoded( + method: .wizardCancel, + params: ["sessionId": AnyCodable(sessionId)]) + self.applyStatusResult(res) + } catch { + self.status = "error" + self.errorMessage = error.localizedDescription + onboardingWizardLogger.error("cancel failed: \(error.localizedDescription, privacy: .public)") + } + } + + private func applyStartResult(_ res: WizardStartResult) { + self.sessionId = res.sessionid + self.status = wizardStatusString(res.status) ?? (res.done ? "done" : "running") + self.errorMessage = res.error + self.currentStep = decodeWizardStep(res.step) + if self.currentStep == nil, res.step != nil { + onboardingWizardLogger.error("wizard step decode failed") + } + if res.done { self.currentStep = nil } + self.restartAttempts = 0 + } + + private func applyNextResult(_ res: WizardNextResult) { + let status = wizardStatusString(res.status) + self.status = status ?? self.status + self.errorMessage = res.error + self.currentStep = decodeWizardStep(res.step) + if self.currentStep == nil, res.step != nil { + onboardingWizardLogger.error("wizard step decode failed") + } + if res.done { self.currentStep = nil } + if res.done || status == "done" || status == "cancelled" || status == "error" { + self.sessionId = nil + } + } + + private func applyStatusResult(_ res: WizardStatusResult) { + self.status = wizardStatusString(res.status) ?? "unknown" + self.errorMessage = res.error + self.currentStep = nil + self.sessionId = nil + } + + private func restartIfSessionLost(error: Error) -> Bool { + guard let gatewayError = error as? GatewayResponseError else { return false } + guard gatewayError.code == ErrorCode.invalidRequest.rawValue else { return false } + let message = gatewayError.message.lowercased() + guard message.contains("wizard not found") || message.contains("wizard not running") else { return false } + guard let mode = self.lastStartMode, self.restartAttempts < self.maxRestartAttempts else { + return false + } + self.restartAttempts += 1 + self.sessionId = nil + self.currentStep = nil + self.status = nil + self.errorMessage = "Wizard session lost. Restarting…" + Task { await self.startIfNeeded(mode: mode, workspace: self.lastStartWorkspace) } + return true + } + + private func shouldSkipWizard() -> Bool { + let root = OpenClawConfigFile.loadDict() + if let wizard = root["wizard"] as? [String: Any], !wizard.isEmpty { + return true + } + if let gateway = root["gateway"] as? [String: Any], + let auth = gateway["auth"] as? [String: Any] + { + if let mode = auth["mode"] as? String, + !mode.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty + { + return true + } + if let token = auth["token"] as? String, + !token.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty + { + return true + } + if let password = auth["password"] as? String, + !password.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty + { + return true + } + } + return false + } +} + +struct OnboardingWizardStepView: View { + let step: WizardStep + let isSubmitting: Bool + let onStepSubmit: (AnyCodable?) -> Void + + @State private var textValue: String + @State private var confirmValue: Bool + @State private var selectedIndex: Int + @State private var selectedIndices: Set + + private let optionItems: [WizardOptionItem] + + init(step: WizardStep, isSubmitting: Bool, onSubmit: @escaping (AnyCodable?) -> Void) { + self.step = step + self.isSubmitting = isSubmitting + self.onStepSubmit = onSubmit + let options = parseWizardOptions(step.options).enumerated().map { index, option in + WizardOptionItem(index: index, option: option) + } + self.optionItems = options + let initialText = anyCodableString(step.initialvalue) + let initialConfirm = anyCodableBool(step.initialvalue) + let initialIndex = options.firstIndex(where: { anyCodableEqual($0.option.value, step.initialvalue) }) ?? 0 + let initialMulti = Set( + options.filter { option in + anyCodableArray(step.initialvalue).contains { anyCodableEqual($0, option.option.value) } + }.map(\.index)) + + _textValue = State(initialValue: initialText) + _confirmValue = State(initialValue: initialConfirm) + _selectedIndex = State(initialValue: initialIndex) + _selectedIndices = State(initialValue: initialMulti) + } + + var body: some View { + VStack(alignment: .leading, spacing: 12) { + if let title = step.title, !title.isEmpty { + Text(title) + .font(.title2.weight(.semibold)) + } + if let message = step.message, !message.isEmpty { + Text(message) + .font(.body) + .foregroundStyle(.secondary) + .fixedSize(horizontal: false, vertical: true) + } + + switch wizardStepType(self.step) { + case "note": + EmptyView() + case "text": + self.textField + case "confirm": + Toggle("", isOn: self.$confirmValue) + .toggleStyle(.switch) + case "select": + self.selectOptions + case "multiselect": + self.multiselectOptions + case "progress": + ProgressView() + .controlSize(.small) + case "action": + EmptyView() + default: + Text("Unsupported step type") + .foregroundStyle(.secondary) + } + + Button(action: self.submit) { + Text(wizardStepType(self.step) == "action" ? "Run" : "Continue") + .frame(minWidth: 120) + } + .buttonStyle(.borderedProminent) + .disabled(self.isSubmitting || self.isBlocked) + } + .frame(maxWidth: .infinity, alignment: .leading) + } + + @ViewBuilder + private var textField: some View { + let isSensitive = self.step.sensitive == true + if isSensitive { + SecureField(self.step.placeholder ?? "", text: self.$textValue) + .textFieldStyle(.roundedBorder) + .frame(maxWidth: 360) + } else { + TextField(self.step.placeholder ?? "", text: self.$textValue) + .textFieldStyle(.roundedBorder) + .frame(maxWidth: 360) + } + } + + private var selectOptions: some View { + VStack(alignment: .leading, spacing: 8) { + ForEach(self.optionItems, id: \.index) { item in + self.selectOptionRow(item) + } + } + } + + private var multiselectOptions: some View { + VStack(alignment: .leading, spacing: 8) { + ForEach(self.optionItems, id: \.index) { item in + self.multiselectOptionRow(item) + } + } + } + + private func selectOptionRow(_ item: WizardOptionItem) -> some View { + Button { + self.selectedIndex = item.index + } label: { + HStack(alignment: .top, spacing: 8) { + Image(systemName: self.selectedIndex == item.index ? "largecircle.fill.circle" : "circle") + .foregroundStyle(Color.accentColor) + VStack(alignment: .leading, spacing: 2) { + Text(item.option.label) + .foregroundStyle(.primary) + if let hint = item.option.hint, !hint.isEmpty { + Text(hint) + .font(.caption) + .foregroundStyle(.secondary) + } + } + } + } + .buttonStyle(.plain) + } + + private func multiselectOptionRow(_ item: WizardOptionItem) -> some View { + Toggle(isOn: self.bindingForOption(item)) { + VStack(alignment: .leading, spacing: 2) { + Text(item.option.label) + if let hint = item.option.hint, !hint.isEmpty { + Text(hint) + .font(.caption) + .foregroundStyle(.secondary) + } + } + } + } + + private func bindingForOption(_ item: WizardOptionItem) -> Binding { + Binding(get: { + self.selectedIndices.contains(item.index) + }, set: { newValue in + if newValue { + self.selectedIndices.insert(item.index) + } else { + self.selectedIndices.remove(item.index) + } + }) + } + + private var isBlocked: Bool { + let type = wizardStepType(step) + if type == "select" { return self.optionItems.isEmpty } + if type == "multiselect" { return self.optionItems.isEmpty } + return false + } + + private func submit() { + switch wizardStepType(self.step) { + case "note", "progress": + self.onStepSubmit(nil) + case "text": + self.onStepSubmit(AnyCodable(self.textValue)) + case "confirm": + self.onStepSubmit(AnyCodable(self.confirmValue)) + case "select": + guard self.optionItems.indices.contains(self.selectedIndex) else { + self.onStepSubmit(nil) + return + } + let option = self.optionItems[self.selectedIndex].option + self.onStepSubmit(bridgeToLocal(option.value) ?? AnyCodable(option.label)) + case "multiselect": + let values = self.optionItems + .filter { self.selectedIndices.contains($0.index) } + .map { bridgeToLocal($0.option.value) ?? AnyCodable($0.option.label) } + self.onStepSubmit(AnyCodable(values)) + case "action": + self.onStepSubmit(AnyCodable(true)) + default: + self.onStepSubmit(nil) + } + } +} + +private struct WizardOptionItem: Identifiable { + let index: Int + let option: WizardOption + + var id: Int { self.index } +} diff --git a/apps/macos/Sources/OpenClaw/OpenClawConfigFile.swift b/apps/macos/Sources/OpenClaw/OpenClawConfigFile.swift new file mode 100644 index 0000000000000000000000000000000000000000..3f7d3c03aa5c7ff95b16b6c6762d2d727af434ed --- /dev/null +++ b/apps/macos/Sources/OpenClaw/OpenClawConfigFile.swift @@ -0,0 +1,217 @@ +import OpenClawProtocol +import Foundation + +enum OpenClawConfigFile { + private static let logger = Logger(subsystem: "ai.openclaw", category: "config") + + static func url() -> URL { + OpenClawPaths.configURL + } + + static func stateDirURL() -> URL { + OpenClawPaths.stateDirURL + } + + static func defaultWorkspaceURL() -> URL { + OpenClawPaths.workspaceURL + } + + static func loadDict() -> [String: Any] { + let url = self.url() + guard FileManager().fileExists(atPath: url.path) else { return [:] } + do { + let data = try Data(contentsOf: url) + guard let root = self.parseConfigData(data) else { + self.logger.warning("config JSON root invalid") + return [:] + } + return root + } catch { + self.logger.warning("config read failed: \(error.localizedDescription)") + return [:] + } + } + + static func saveDict(_ dict: [String: Any]) { + // Nix mode disables config writes in production, but tests rely on saving temp configs. + if ProcessInfo.processInfo.isNixMode, !ProcessInfo.processInfo.isRunningTests { return } + do { + let data = try JSONSerialization.data(withJSONObject: dict, options: [.prettyPrinted, .sortedKeys]) + let url = self.url() + try FileManager().createDirectory( + at: url.deletingLastPathComponent(), + withIntermediateDirectories: true) + try data.write(to: url, options: [.atomic]) + } catch { + self.logger.error("config save failed: \(error.localizedDescription)") + } + } + + static func loadGatewayDict() -> [String: Any] { + let root = self.loadDict() + return root["gateway"] as? [String: Any] ?? [:] + } + + static func updateGatewayDict(_ mutate: (inout [String: Any]) -> Void) { + var root = self.loadDict() + var gateway = root["gateway"] as? [String: Any] ?? [:] + mutate(&gateway) + if gateway.isEmpty { + root.removeValue(forKey: "gateway") + } else { + root["gateway"] = gateway + } + self.saveDict(root) + } + + static func browserControlEnabled(defaultValue: Bool = true) -> Bool { + let root = self.loadDict() + let browser = root["browser"] as? [String: Any] + return browser?["enabled"] as? Bool ?? defaultValue + } + + static func setBrowserControlEnabled(_ enabled: Bool) { + var root = self.loadDict() + var browser = root["browser"] as? [String: Any] ?? [:] + browser["enabled"] = enabled + root["browser"] = browser + self.saveDict(root) + self.logger.debug("browser control updated enabled=\(enabled)") + } + + static func agentWorkspace() -> String? { + let root = self.loadDict() + let agents = root["agents"] as? [String: Any] + let defaults = agents?["defaults"] as? [String: Any] + return defaults?["workspace"] as? String + } + + static func setAgentWorkspace(_ workspace: String?) { + var root = self.loadDict() + var agents = root["agents"] as? [String: Any] ?? [:] + var defaults = agents["defaults"] as? [String: Any] ?? [:] + let trimmed = workspace?.trimmingCharacters(in: .whitespacesAndNewlines) ?? "" + if trimmed.isEmpty { + defaults.removeValue(forKey: "workspace") + } else { + defaults["workspace"] = trimmed + } + if defaults.isEmpty { + agents.removeValue(forKey: "defaults") + } else { + agents["defaults"] = defaults + } + if agents.isEmpty { + root.removeValue(forKey: "agents") + } else { + root["agents"] = agents + } + self.saveDict(root) + self.logger.debug("agents.defaults.workspace updated set=\(!trimmed.isEmpty)") + } + + static func gatewayPassword() -> String? { + let root = self.loadDict() + guard let gateway = root["gateway"] as? [String: Any], + let remote = gateway["remote"] as? [String: Any] + else { + return nil + } + return remote["password"] as? String + } + + static func gatewayPort() -> Int? { + let root = self.loadDict() + guard let gateway = root["gateway"] as? [String: Any] else { return nil } + if let port = gateway["port"] as? Int, port > 0 { return port } + if let number = gateway["port"] as? NSNumber, number.intValue > 0 { + return number.intValue + } + if let raw = gateway["port"] as? String, + let parsed = Int(raw.trimmingCharacters(in: .whitespacesAndNewlines)), + parsed > 0 + { + return parsed + } + return nil + } + + static func remoteGatewayPort() -> Int? { + guard let url = self.remoteGatewayUrl(), + let port = url.port, + port > 0 + else { return nil } + return port + } + + static func remoteGatewayPort(matchingHost sshHost: String) -> Int? { + let trimmedSshHost = sshHost.trimmingCharacters(in: .whitespacesAndNewlines) + guard !trimmedSshHost.isEmpty, + let url = self.remoteGatewayUrl(), + let port = url.port, + port > 0, + let urlHost = url.host?.trimmingCharacters(in: .whitespacesAndNewlines), + !urlHost.isEmpty + else { + return nil + } + + let sshKey = Self.hostKey(trimmedSshHost) + let urlKey = Self.hostKey(urlHost) + guard !sshKey.isEmpty, !urlKey.isEmpty, sshKey == urlKey else { return nil } + return port + } + + static func setRemoteGatewayUrl(host: String, port: Int?) { + guard let port, port > 0 else { return } + let trimmedHost = host.trimmingCharacters(in: .whitespacesAndNewlines) + guard !trimmedHost.isEmpty else { return } + self.updateGatewayDict { gateway in + var remote = gateway["remote"] as? [String: Any] ?? [:] + let existingUrl = (remote["url"] as? String)? + .trimmingCharacters(in: .whitespacesAndNewlines) ?? "" + let scheme = URL(string: existingUrl)?.scheme ?? "ws" + remote["url"] = "\(scheme)://\(trimmedHost):\(port)" + gateway["remote"] = remote + } + } + + private static func remoteGatewayUrl() -> URL? { + let root = self.loadDict() + guard let gateway = root["gateway"] as? [String: Any], + let remote = gateway["remote"] as? [String: Any], + let raw = remote["url"] as? String + else { + return nil + } + let trimmed = raw.trimmingCharacters(in: .whitespacesAndNewlines) + guard !trimmed.isEmpty, let url = URL(string: trimmed) else { return nil } + return url + } + + private static func hostKey(_ host: String) -> String { + let trimmed = host.trimmingCharacters(in: .whitespacesAndNewlines).lowercased() + guard !trimmed.isEmpty else { return "" } + if trimmed.contains(":") { return trimmed } + let digits = CharacterSet(charactersIn: "0123456789.") + if trimmed.rangeOfCharacter(from: digits.inverted) == nil { + return trimmed + } + return trimmed.split(separator: ".").first.map(String.init) ?? trimmed + } + + private static func parseConfigData(_ data: Data) -> [String: Any]? { + if let root = try? JSONSerialization.jsonObject(with: data) as? [String: Any] { + return root + } + let decoder = JSONDecoder() + if #available(macOS 12.0, *) { + decoder.allowsJSON5 = true + } + if let decoded = try? decoder.decode([String: AnyCodable].self, from: data) { + self.logger.notice("config parsed with JSON5 decoder") + return decoded.mapValues { $0.foundationValue } + } + return nil + } +} diff --git a/apps/macos/Sources/OpenClaw/OpenClawPaths.swift b/apps/macos/Sources/OpenClaw/OpenClawPaths.swift new file mode 100644 index 0000000000000000000000000000000000000000..632c07c802bdf2a97792db6769ea268610bc46f7 --- /dev/null +++ b/apps/macos/Sources/OpenClaw/OpenClawPaths.swift @@ -0,0 +1,54 @@ +import Foundation + +enum OpenClawEnv { + static func path(_ key: String) -> String? { + // Normalize env overrides once so UI + file IO stay consistent. + guard let raw = getenv(key) else { return nil } + let value = String(cString: raw).trimmingCharacters(in: .whitespacesAndNewlines) + guard !value.isEmpty + else { + return nil + } + return value + } +} + +enum OpenClawPaths { + private static let configPathEnv = ["OPENCLAW_CONFIG_PATH"] + private static let stateDirEnv = ["OPENCLAW_STATE_DIR"] + + static var stateDirURL: URL { + for key in self.stateDirEnv { + if let override = OpenClawEnv.path(key) { + return URL(fileURLWithPath: override, isDirectory: true) + } + } + let home = FileManager().homeDirectoryForCurrentUser + let preferred = home.appendingPathComponent(".openclaw", isDirectory: true) + return preferred + } + + private static func resolveConfigCandidate(in dir: URL) -> URL? { + let candidates = [ + dir.appendingPathComponent("openclaw.json"), + ] + return candidates.first(where: { FileManager().fileExists(atPath: $0.path) }) + } + + static var configURL: URL { + for key in self.configPathEnv { + if let override = OpenClawEnv.path(key) { + return URL(fileURLWithPath: override) + } + } + let stateDir = self.stateDirURL + if let existing = self.resolveConfigCandidate(in: stateDir) { + return existing + } + return stateDir.appendingPathComponent("openclaw.json") + } + + static var workspaceURL: URL { + self.stateDirURL.appendingPathComponent("workspace", isDirectory: true) + } +} diff --git a/apps/macos/Sources/OpenClaw/PeekabooBridgeHostCoordinator.swift b/apps/macos/Sources/OpenClaw/PeekabooBridgeHostCoordinator.swift new file mode 100644 index 0000000000000000000000000000000000000000..9f97650b9f2cd110a19ac5367d2a630f313a7c5b --- /dev/null +++ b/apps/macos/Sources/OpenClaw/PeekabooBridgeHostCoordinator.swift @@ -0,0 +1,137 @@ +import Foundation +import os +import PeekabooAutomationKit +import PeekabooBridge +import PeekabooFoundation +import Security + +@MainActor +final class PeekabooBridgeHostCoordinator { + static let shared = PeekabooBridgeHostCoordinator() + + private let logger = Logger(subsystem: "ai.openclaw", category: "PeekabooBridge") + + private var host: PeekabooBridgeHost? + private var services: OpenClawPeekabooBridgeServices? + private static var openclawSocketPath: String { + let fileManager = FileManager.default + let base = fileManager.urls(for: .applicationSupportDirectory, in: .userDomainMask).first + ?? fileManager.homeDirectoryForCurrentUser.appendingPathComponent("Library/Application Support") + let directory = base.appendingPathComponent("OpenClaw", isDirectory: true) + return directory.appendingPathComponent(PeekabooBridgeConstants.socketName, isDirectory: false).path + } + + func setEnabled(_ enabled: Bool) async { + if enabled { + await self.startIfNeeded() + } else { + await self.stop() + } + } + + func stop() async { + guard let host else { return } + await host.stop() + self.host = nil + self.services = nil + self.logger.info("PeekabooBridge host stopped") + } + + private func startIfNeeded() async { + guard self.host == nil else { return } + + var allowlistedTeamIDs: Set = ["Y5PE65HELJ"] + if let teamID = Self.currentTeamID() { + allowlistedTeamIDs.insert(teamID) + } + let allowlistedBundles: Set = [] + + let services = OpenClawPeekabooBridgeServices() + let server = PeekabooBridgeServer( + services: services, + hostKind: .gui, + allowlistedTeams: allowlistedTeamIDs, + allowlistedBundles: allowlistedBundles) + + let host = PeekabooBridgeHost( + socketPath: Self.openclawSocketPath, + server: server, + allowedTeamIDs: allowlistedTeamIDs, + requestTimeoutSec: 10) + + self.services = services + self.host = host + + await host.start() + self.logger + .info("PeekabooBridge host started at \(Self.openclawSocketPath, privacy: .public)") + } + + private static func currentTeamID() -> String? { + var code: SecCode? + guard SecCodeCopySelf(SecCSFlags(), &code) == errSecSuccess, + let code + else { + return nil + } + + var staticCode: SecStaticCode? + guard SecCodeCopyStaticCode(code, SecCSFlags(), &staticCode) == errSecSuccess, + let staticCode + else { + return nil + } + + var infoCF: CFDictionary? + guard SecCodeCopySigningInformation( + staticCode, + SecCSFlags(rawValue: kSecCSSigningInformation), + &infoCF) == errSecSuccess, + let info = infoCF as? [String: Any] + else { + return nil + } + + return info[kSecCodeInfoTeamIdentifier as String] as? String + } +} + +@MainActor +private final class OpenClawPeekabooBridgeServices: PeekabooBridgeServiceProviding { + let permissions: PermissionsService + let screenCapture: any ScreenCaptureServiceProtocol + let automation: any UIAutomationServiceProtocol + let windows: any WindowManagementServiceProtocol + let applications: any ApplicationServiceProtocol + let menu: any MenuServiceProtocol + let dock: any DockServiceProtocol + let dialogs: any DialogServiceProtocol + let snapshots: any SnapshotManagerProtocol + + init() { + let logging = LoggingService(subsystem: "ai.openclaw.peekaboo") + let feedbackClient: any AutomationFeedbackClient = NoopAutomationFeedbackClient() + + let snapshots = InMemorySnapshotManager(options: .init( + snapshotValidityWindow: 600, + maxSnapshots: 50, + deleteArtifactsOnCleanup: false)) + let applications = ApplicationService(feedbackClient: feedbackClient) + + let screenCapture = ScreenCaptureService(loggingService: logging) + + self.permissions = PermissionsService() + self.snapshots = snapshots + self.applications = applications + self.screenCapture = screenCapture + self.automation = UIAutomationService( + snapshotManager: snapshots, + loggingService: logging, + searchPolicy: .balanced, + feedbackClient: feedbackClient) + self.windows = WindowManagementService(applicationService: applications, feedbackClient: feedbackClient) + self.menu = MenuService(applicationService: applications, feedbackClient: feedbackClient) + self.dock = DockService(feedbackClient: feedbackClient) + self.dialogs = DialogService(feedbackClient: feedbackClient) + } +} diff --git a/apps/macos/Sources/OpenClaw/PermissionManager.swift b/apps/macos/Sources/OpenClaw/PermissionManager.swift new file mode 100644 index 0000000000000000000000000000000000000000..3cf1cba3f6ec8edd43120083c302e3837b03b590 --- /dev/null +++ b/apps/macos/Sources/OpenClaw/PermissionManager.swift @@ -0,0 +1,506 @@ +import AppKit +import ApplicationServices +import AVFoundation +import OpenClawIPC +import CoreGraphics +import CoreLocation +import Foundation +import Observation +import Speech +import UserNotifications + +enum PermissionManager { + static func isLocationAuthorized(status: CLAuthorizationStatus, requireAlways: Bool) -> Bool { + if requireAlways { return status == .authorizedAlways } + switch status { + case .authorizedAlways, .authorizedWhenInUse: + return true + case .authorized: // deprecated, but still shows up on some macOS versions + return true + default: + return false + } + } + + static func ensure(_ caps: [Capability], interactive: Bool) async -> [Capability: Bool] { + var results: [Capability: Bool] = [:] + for cap in caps { + results[cap] = await self.ensureCapability(cap, interactive: interactive) + } + return results + } + + private static func ensureCapability(_ cap: Capability, interactive: Bool) async -> Bool { + switch cap { + case .notifications: + await self.ensureNotifications(interactive: interactive) + case .appleScript: + await self.ensureAppleScript(interactive: interactive) + case .accessibility: + await self.ensureAccessibility(interactive: interactive) + case .screenRecording: + await self.ensureScreenRecording(interactive: interactive) + case .microphone: + await self.ensureMicrophone(interactive: interactive) + case .speechRecognition: + await self.ensureSpeechRecognition(interactive: interactive) + case .camera: + await self.ensureCamera(interactive: interactive) + case .location: + await self.ensureLocation(interactive: interactive) + } + } + + private static func ensureNotifications(interactive: Bool) async -> Bool { + let center = UNUserNotificationCenter.current() + let settings = await center.notificationSettings() + + switch settings.authorizationStatus { + case .authorized, .provisional, .ephemeral: + return true + case .notDetermined: + guard interactive else { return false } + let granted = await (try? center.requestAuthorization(options: [.alert, .sound, .badge])) ?? false + let updated = await center.notificationSettings() + return granted && + (updated.authorizationStatus == .authorized || updated.authorizationStatus == .provisional) + case .denied: + if interactive { + NotificationPermissionHelper.openSettings() + } + return false + @unknown default: + return false + } + } + + private static func ensureAppleScript(interactive: Bool) async -> Bool { + let granted = await MainActor.run { AppleScriptPermission.isAuthorized() } + if interactive, !granted { + await AppleScriptPermission.requestAuthorization() + } + return await MainActor.run { AppleScriptPermission.isAuthorized() } + } + + private static func ensureAccessibility(interactive: Bool) async -> Bool { + let trusted = await MainActor.run { AXIsProcessTrusted() } + if interactive, !trusted { + await MainActor.run { + let opts: NSDictionary = ["AXTrustedCheckOptionPrompt": true] + _ = AXIsProcessTrustedWithOptions(opts) + } + } + return await MainActor.run { AXIsProcessTrusted() } + } + + private static func ensureScreenRecording(interactive: Bool) async -> Bool { + let granted = ScreenRecordingProbe.isAuthorized() + if interactive, !granted { + await ScreenRecordingProbe.requestAuthorization() + } + return ScreenRecordingProbe.isAuthorized() + } + + private static func ensureMicrophone(interactive: Bool) async -> Bool { + let status = AVCaptureDevice.authorizationStatus(for: .audio) + switch status { + case .authorized: + return true + case .notDetermined: + guard interactive else { return false } + return await AVCaptureDevice.requestAccess(for: .audio) + case .denied, .restricted: + if interactive { + MicrophonePermissionHelper.openSettings() + } + return false + @unknown default: + return false + } + } + + private static func ensureSpeechRecognition(interactive: Bool) async -> Bool { + let status = SFSpeechRecognizer.authorizationStatus() + if status == .notDetermined, interactive { + await withUnsafeContinuation { (cont: UnsafeContinuation) in + SFSpeechRecognizer.requestAuthorization { _ in + DispatchQueue.main.async { cont.resume() } + } + } + } + return SFSpeechRecognizer.authorizationStatus() == .authorized + } + + private static func ensureCamera(interactive: Bool) async -> Bool { + let status = AVCaptureDevice.authorizationStatus(for: .video) + switch status { + case .authorized: + return true + case .notDetermined: + guard interactive else { return false } + return await AVCaptureDevice.requestAccess(for: .video) + case .denied, .restricted: + if interactive { + CameraPermissionHelper.openSettings() + } + return false + @unknown default: + return false + } + } + + private static func ensureLocation(interactive: Bool) async -> Bool { + guard CLLocationManager.locationServicesEnabled() else { + if interactive { + await MainActor.run { LocationPermissionHelper.openSettings() } + } + return false + } + let status = CLLocationManager().authorizationStatus + switch status { + case .authorizedAlways, .authorizedWhenInUse, .authorized: + return true + case .notDetermined: + guard interactive else { return false } + let updated = await LocationPermissionRequester.shared.request(always: false) + return self.isLocationAuthorized(status: updated, requireAlways: false) + case .denied, .restricted: + if interactive { + await MainActor.run { LocationPermissionHelper.openSettings() } + } + return false + @unknown default: + return false + } + } + + static func voiceWakePermissionsGranted() -> Bool { + let mic = AVCaptureDevice.authorizationStatus(for: .audio) == .authorized + let speech = SFSpeechRecognizer.authorizationStatus() == .authorized + return mic && speech + } + + static func ensureVoiceWakePermissions(interactive: Bool) async -> Bool { + let results = await self.ensure([.microphone, .speechRecognition], interactive: interactive) + return results[.microphone] == true && results[.speechRecognition] == true + } + + static func status(_ caps: [Capability] = Capability.allCases) async -> [Capability: Bool] { + var results: [Capability: Bool] = [:] + for cap in caps { + switch cap { + case .notifications: + let center = UNUserNotificationCenter.current() + let settings = await center.notificationSettings() + results[cap] = settings.authorizationStatus == .authorized + || settings.authorizationStatus == .provisional + + case .appleScript: + results[cap] = await MainActor.run { AppleScriptPermission.isAuthorized() } + + case .accessibility: + results[cap] = await MainActor.run { AXIsProcessTrusted() } + + case .screenRecording: + if #available(macOS 10.15, *) { + results[cap] = CGPreflightScreenCaptureAccess() + } else { + results[cap] = true + } + + case .microphone: + results[cap] = AVCaptureDevice.authorizationStatus(for: .audio) == .authorized + + case .speechRecognition: + results[cap] = SFSpeechRecognizer.authorizationStatus() == .authorized + + case .camera: + results[cap] = AVCaptureDevice.authorizationStatus(for: .video) == .authorized + + case .location: + let status = CLLocationManager().authorizationStatus + results[cap] = CLLocationManager.locationServicesEnabled() + && self.isLocationAuthorized(status: status, requireAlways: false) + } + } + return results + } +} + +enum NotificationPermissionHelper { + static func openSettings() { + let candidates = [ + "x-apple.systempreferences:com.apple.Notifications-Settings.extension", + "x-apple.systempreferences:com.apple.preference.notifications", + ] + + for candidate in candidates { + if let url = URL(string: candidate), NSWorkspace.shared.open(url) { + return + } + } + } +} + +enum MicrophonePermissionHelper { + static func openSettings() { + let candidates = [ + "x-apple.systempreferences:com.apple.preference.security?Privacy_Microphone", + "x-apple.systempreferences:com.apple.preference.security", + ] + + for candidate in candidates { + if let url = URL(string: candidate), NSWorkspace.shared.open(url) { + return + } + } + } +} + +enum CameraPermissionHelper { + static func openSettings() { + let candidates = [ + "x-apple.systempreferences:com.apple.preference.security?Privacy_Camera", + "x-apple.systempreferences:com.apple.preference.security", + ] + + for candidate in candidates { + if let url = URL(string: candidate), NSWorkspace.shared.open(url) { + return + } + } + } +} + +enum LocationPermissionHelper { + static func openSettings() { + let candidates = [ + "x-apple.systempreferences:com.apple.preference.security?Privacy_LocationServices", + "x-apple.systempreferences:com.apple.preference.security", + ] + + for candidate in candidates { + if let url = URL(string: candidate), NSWorkspace.shared.open(url) { + return + } + } + } +} + +@MainActor +final class LocationPermissionRequester: NSObject, CLLocationManagerDelegate { + static let shared = LocationPermissionRequester() + private let manager = CLLocationManager() + private var continuation: CheckedContinuation? + private var timeoutTask: Task? + + override init() { + super.init() + self.manager.delegate = self + } + + func request(always: Bool) async -> CLAuthorizationStatus { + let current = self.manager.authorizationStatus + if PermissionManager.isLocationAuthorized(status: current, requireAlways: always) { + return current + } + + return await withCheckedContinuation { cont in + self.continuation = cont + self.timeoutTask?.cancel() + self.timeoutTask = Task { [weak self] in + try? await Task.sleep(nanoseconds: 3_000_000_000) + await MainActor.run { [weak self] in + guard let self else { return } + guard self.continuation != nil else { return } + LocationPermissionHelper.openSettings() + self.finish(status: self.manager.authorizationStatus) + } + } + if always { + self.manager.requestAlwaysAuthorization() + } else { + self.manager.requestWhenInUseAuthorization() + } + + // On macOS, requesting an actual fix makes the prompt more reliable. + self.manager.requestLocation() + } + } + + private func finish(status: CLAuthorizationStatus) { + self.timeoutTask?.cancel() + self.timeoutTask = nil + guard let cont = self.continuation else { return } + self.continuation = nil + cont.resume(returning: status) + } + + // nonisolated for Swift 6 strict concurrency compatibility + nonisolated func locationManagerDidChangeAuthorization(_ manager: CLLocationManager) { + let status = manager.authorizationStatus + Task { @MainActor in + self.finish(status: status) + } + } + + // Legacy callback (still used on some macOS versions / configurations). + nonisolated func locationManager( + _ manager: CLLocationManager, + didChangeAuthorization status: CLAuthorizationStatus) + { + Task { @MainActor in + self.finish(status: status) + } + } + + nonisolated func locationManager(_ manager: CLLocationManager, didFailWithError error: Error) { + let status = manager.authorizationStatus + Task { @MainActor in + if status == .denied || status == .restricted { + LocationPermissionHelper.openSettings() + } + self.finish(status: status) + } + } + + nonisolated func locationManager(_ manager: CLLocationManager, didUpdateLocations locations: [CLLocation]) { + let status = manager.authorizationStatus + Task { @MainActor in + self.finish(status: status) + } + } +} + +enum AppleScriptPermission { + private static let logger = Logger(subsystem: "ai.openclaw", category: "AppleScriptPermission") + + /// Sends a benign AppleScript to Terminal to verify Automation permission. + @MainActor + static func isAuthorized() -> Bool { + let script = """ + tell application "Terminal" + return "openclaw-ok" + end tell + """ + + var error: NSDictionary? + let appleScript = NSAppleScript(source: script) + let result = appleScript?.executeAndReturnError(&error) + + if let error, let code = error["NSAppleScriptErrorNumber"] as? Int { + if code == -1743 { // errAEEventWouldRequireUserConsent + Self.logger.debug("AppleScript permission denied (-1743)") + return false + } + Self.logger.debug("AppleScript check failed with code \(code)") + } + + return result != nil + } + + /// Triggers the TCC prompt and opens System Settings → Privacy & Security → Automation. + @MainActor + static func requestAuthorization() async { + _ = self.isAuthorized() // first attempt triggers the dialog if not granted + + // Open the Automation pane to help the user if the prompt was dismissed. + let urlStrings = [ + "x-apple.systempreferences:com.apple.preference.security?Privacy_Automation", + "x-apple.systempreferences:com.apple.preference.security", + ] + + for candidate in urlStrings { + if let url = URL(string: candidate), NSWorkspace.shared.open(url) { + break + } + } + } +} + +@MainActor +@Observable +final class PermissionMonitor { + static let shared = PermissionMonitor() + + private(set) var status: [Capability: Bool] = [:] + + private var monitorTimer: Timer? + private var isChecking = false + private var registrations = 0 + private var lastCheck: Date? + private let minimumCheckInterval: TimeInterval = 0.5 + + func register() { + self.registrations += 1 + if self.registrations == 1 { + self.startMonitoring() + } + } + + func unregister() { + guard self.registrations > 0 else { return } + self.registrations -= 1 + if self.registrations == 0 { + self.stopMonitoring() + } + } + + func refreshNow() async { + await self.checkStatus(force: true) + } + + private func startMonitoring() { + Task { await self.checkStatus(force: true) } + + if ProcessInfo.processInfo.isRunningTests { + return + } + self.monitorTimer = Timer.scheduledTimer(withTimeInterval: 1.0, repeats: true) { [weak self] _ in + guard let self else { return } + Task { @MainActor in + await self.checkStatus(force: false) + } + } + } + + private func stopMonitoring() { + self.monitorTimer?.invalidate() + self.monitorTimer = nil + self.lastCheck = nil + } + + private func checkStatus(force: Bool) async { + if self.isChecking { return } + let now = Date() + if !force, let lastCheck, now.timeIntervalSince(lastCheck) < self.minimumCheckInterval { + return + } + + self.isChecking = true + + let latest = await PermissionManager.status() + if latest != self.status { + self.status = latest + } + self.lastCheck = Date() + + self.isChecking = false + } +} + +enum ScreenRecordingProbe { + static func isAuthorized() -> Bool { + if #available(macOS 10.15, *) { + return CGPreflightScreenCaptureAccess() + } + return true + } + + @MainActor + static func requestAuthorization() async { + if #available(macOS 10.15, *) { + _ = CGRequestScreenCaptureAccess() + } + } +} diff --git a/apps/macos/Sources/OpenClaw/PermissionsSettings.swift b/apps/macos/Sources/OpenClaw/PermissionsSettings.swift new file mode 100644 index 0000000000000000000000000000000000000000..a8f6accf8af7bf431da965893ea8b054c9ce57ff --- /dev/null +++ b/apps/macos/Sources/OpenClaw/PermissionsSettings.swift @@ -0,0 +1,227 @@ +import OpenClawIPC +import OpenClawKit +import CoreLocation +import SwiftUI + +struct PermissionsSettings: View { + let status: [Capability: Bool] + let refresh: () async -> Void + let showOnboarding: () -> Void + + var body: some View { + VStack(alignment: .leading, spacing: 14) { + SystemRunSettingsView() + + Text("Allow these so OpenClaw can notify and capture when needed.") + .padding(.top, 4) + + PermissionStatusList(status: self.status, refresh: self.refresh) + .padding(.horizontal, 2) + .padding(.vertical, 6) + + LocationAccessSettings() + + Button("Restart onboarding") { self.showOnboarding() } + .buttonStyle(.bordered) + Spacer() + } + .frame(maxWidth: .infinity, alignment: .leading) + .padding(.horizontal, 12) + } +} + +private struct LocationAccessSettings: View { + @AppStorage(locationModeKey) private var locationModeRaw: String = OpenClawLocationMode.off.rawValue + @AppStorage(locationPreciseKey) private var locationPreciseEnabled: Bool = true + @State private var lastLocationModeRaw: String = OpenClawLocationMode.off.rawValue + + var body: some View { + VStack(alignment: .leading, spacing: 6) { + Text("Location Access") + .font(.body) + + Picker("", selection: self.$locationModeRaw) { + Text("Off").tag(OpenClawLocationMode.off.rawValue) + Text("While Using").tag(OpenClawLocationMode.whileUsing.rawValue) + Text("Always").tag(OpenClawLocationMode.always.rawValue) + } + .labelsHidden() + .pickerStyle(.menu) + + Toggle("Precise Location", isOn: self.$locationPreciseEnabled) + .disabled(self.locationMode == .off) + + Text("Always may require System Settings to approve background location.") + .font(.footnote) + .foregroundStyle(.tertiary) + .fixedSize(horizontal: false, vertical: true) + } + .onAppear { + self.lastLocationModeRaw = self.locationModeRaw + } + .onChange(of: self.locationModeRaw) { _, newValue in + let previous = self.lastLocationModeRaw + self.lastLocationModeRaw = newValue + guard let mode = OpenClawLocationMode(rawValue: newValue) else { return } + Task { + let granted = await self.requestLocationAuthorization(mode: mode) + if !granted { + await MainActor.run { + self.locationModeRaw = previous + self.lastLocationModeRaw = previous + } + } + } + } + } + + private var locationMode: OpenClawLocationMode { + OpenClawLocationMode(rawValue: self.locationModeRaw) ?? .off + } + + private func requestLocationAuthorization(mode: OpenClawLocationMode) async -> Bool { + guard mode != .off else { return true } + guard CLLocationManager.locationServicesEnabled() else { + await MainActor.run { LocationPermissionHelper.openSettings() } + return false + } + + let status = CLLocationManager().authorizationStatus + let requireAlways = mode == .always + if PermissionManager.isLocationAuthorized(status: status, requireAlways: requireAlways) { + return true + } + let updated = await LocationPermissionRequester.shared.request(always: requireAlways) + return PermissionManager.isLocationAuthorized(status: updated, requireAlways: requireAlways) + } +} + +struct PermissionStatusList: View { + let status: [Capability: Bool] + let refresh: () async -> Void + + var body: some View { + VStack(alignment: .leading, spacing: 12) { + ForEach(Capability.allCases, id: \.self) { cap in + PermissionRow(capability: cap, status: self.status[cap] ?? false) { + Task { await self.handle(cap) } + } + } + Button { + Task { await self.refresh() } + } label: { + Label("Refresh", systemImage: "arrow.clockwise") + } + .buttonStyle(.bordered) + .controlSize(.small) + .font(.footnote) + .padding(.top, 2) + .help("Refresh status") + } + } + + @MainActor + private func handle(_ cap: Capability) async { + _ = await PermissionManager.ensure([cap], interactive: true) + await self.refresh() + } +} + +struct PermissionRow: View { + let capability: Capability + let status: Bool + let compact: Bool + let action: () -> Void + + init(capability: Capability, status: Bool, compact: Bool = false, action: @escaping () -> Void) { + self.capability = capability + self.status = status + self.compact = compact + self.action = action + } + + var body: some View { + HStack(spacing: self.compact ? 10 : 12) { + ZStack { + Circle().fill(self.status ? Color.green.opacity(0.2) : Color.gray.opacity(0.15)) + .frame(width: self.iconSize, height: self.iconSize) + Image(systemName: self.icon) + .foregroundStyle(self.status ? Color.green : Color.secondary) + } + VStack(alignment: .leading, spacing: 2) { + Text(self.title).font(.body.weight(.semibold)) + Text(self.subtitle).font(.caption).foregroundStyle(.secondary) + } + Spacer() + if self.status { + Label("Granted", systemImage: "checkmark.circle.fill") + .foregroundStyle(.green) + } else { + Button("Grant") { self.action() } + .buttonStyle(.bordered) + } + } + .padding(.vertical, self.compact ? 4 : 6) + } + + private var iconSize: CGFloat { self.compact ? 28 : 32 } + + private var title: String { + switch self.capability { + case .appleScript: "Automation (AppleScript)" + case .notifications: "Notifications" + case .accessibility: "Accessibility" + case .screenRecording: "Screen Recording" + case .microphone: "Microphone" + case .speechRecognition: "Speech Recognition" + case .camera: "Camera" + case .location: "Location" + } + } + + private var subtitle: String { + switch self.capability { + case .appleScript: + "Control other apps (e.g. Terminal) for automation actions" + case .notifications: "Show desktop alerts for agent activity" + case .accessibility: "Control UI elements when an action requires it" + case .screenRecording: "Capture the screen for context or screenshots" + case .microphone: "Allow Voice Wake and audio capture" + case .speechRecognition: "Transcribe Voice Wake trigger phrases on-device" + case .camera: "Capture photos and video from the camera" + case .location: "Share location when requested by the agent" + } + } + + private var icon: String { + switch self.capability { + case .appleScript: "applescript" + case .notifications: "bell" + case .accessibility: "hand.raised" + case .screenRecording: "display" + case .microphone: "mic" + case .speechRecognition: "waveform" + case .camera: "camera" + case .location: "location" + } + } +} + +#if DEBUG +struct PermissionsSettings_Previews: PreviewProvider { + static var previews: some View { + PermissionsSettings( + status: [ + .appleScript: true, + .notifications: true, + .accessibility: false, + .screenRecording: false, + .microphone: true, + .speechRecognition: false, + ], + refresh: {}, + showOnboarding: {}) + .frame(width: SettingsTab.windowWidth, height: SettingsTab.windowHeight) + } +} +#endif diff --git a/apps/macos/Sources/OpenClaw/PointingHandCursor.swift b/apps/macos/Sources/OpenClaw/PointingHandCursor.swift new file mode 100644 index 0000000000000000000000000000000000000000..ceb6fb6f81dd4b0dda24bc5718c6bb5a62dcb5a9 --- /dev/null +++ b/apps/macos/Sources/OpenClaw/PointingHandCursor.swift @@ -0,0 +1,30 @@ +import AppKit +import SwiftUI + +private struct PointingHandCursorModifier: ViewModifier { + @State private var isHovering = false + + func body(content: Content) -> some View { + content + .onHover { hovering in + guard hovering != self.isHovering else { return } + self.isHovering = hovering + if hovering { + NSCursor.pointingHand.push() + } else { + NSCursor.pop() + } + } + .onDisappear { + guard self.isHovering else { return } + self.isHovering = false + NSCursor.pop() + } + } +} + +extension View { + func pointingHandCursor() -> some View { + self.modifier(PointingHandCursorModifier()) + } +} diff --git a/apps/macos/Sources/OpenClaw/PortGuardian.swift b/apps/macos/Sources/OpenClaw/PortGuardian.swift new file mode 100644 index 0000000000000000000000000000000000000000..98225f30e1e568f7227469538331f24b9c405d66 --- /dev/null +++ b/apps/macos/Sources/OpenClaw/PortGuardian.swift @@ -0,0 +1,418 @@ +import Foundation +import OSLog +#if canImport(Darwin) +import Darwin +#endif + +actor PortGuardian { + static let shared = PortGuardian() + + struct Record: Codable { + let port: Int + let pid: Int32 + let command: String + let mode: String + let timestamp: TimeInterval + } + + struct Descriptor: Sendable { + let pid: Int32 + let command: String + let executablePath: String? + } + + private var records: [Record] = [] + private let logger = Logger(subsystem: "ai.openclaw", category: "portguard") + private nonisolated static let appSupportDir: URL = { + let base = FileManager().urls(for: .applicationSupportDirectory, in: .userDomainMask).first! + return base.appendingPathComponent("OpenClaw", isDirectory: true) + }() + + private nonisolated static var recordPath: URL { + self.appSupportDir.appendingPathComponent("port-guard.json", isDirectory: false) + } + + init() { + self.records = Self.loadRecords(from: Self.recordPath) + } + + func sweep(mode: AppState.ConnectionMode) async { + self.logger.info("port sweep starting (mode=\(mode.rawValue, privacy: .public))") + guard mode != .unconfigured else { + self.logger.info("port sweep skipped (mode=unconfigured)") + return + } + let ports = [GatewayEnvironment.gatewayPort()] + for port in ports { + let listeners = await self.listeners(on: port) + guard !listeners.isEmpty else { continue } + for listener in listeners { + if self.isExpected(listener, port: port, mode: mode) { + let message = """ + port \(port) already served by expected \(listener.command) + (pid \(listener.pid)) — keeping + """ + self.logger.info("\(message, privacy: .public)") + continue + } + let killed = await self.kill(listener.pid) + if killed { + let message = """ + port \(port) was held by \(listener.command) + (pid \(listener.pid)); terminated + """ + self.logger.error("\(message, privacy: .public)") + } else { + self.logger.error("failed to terminate pid \(listener.pid) on port \(port, privacy: .public)") + } + } + } + self.logger.info("port sweep done") + } + + func record(port: Int, pid: Int32, command: String, mode: AppState.ConnectionMode) async { + try? FileManager().createDirectory(at: Self.appSupportDir, withIntermediateDirectories: true) + self.records.removeAll { $0.pid == pid } + self.records.append( + Record( + port: port, + pid: pid, + command: command, + mode: mode.rawValue, + timestamp: Date().timeIntervalSince1970)) + self.save() + } + + func removeRecord(pid: Int32) { + let before = self.records.count + self.records.removeAll { $0.pid == pid } + if self.records.count != before { + self.save() + } + } + + struct PortReport: Identifiable { + enum Status { + case ok(String) + case missing(String) + case interference(String, offenders: [ReportListener]) + } + + let port: Int + let expected: String + let status: Status + let listeners: [ReportListener] + + var id: Int { self.port } + + var offenders: [ReportListener] { + if case let .interference(_, offenders) = self.status { return offenders } + return [] + } + + var summary: String { + switch self.status { + case let .ok(text): text + case let .missing(text): text + case let .interference(text, _): text + } + } + } + + func describe(port: Int) async -> Descriptor? { + guard let listener = await self.listeners(on: port).first else { return nil } + let path = Self.executablePath(for: listener.pid) + return Descriptor(pid: listener.pid, command: listener.command, executablePath: path) + } + + // MARK: - Internals + + private struct Listener { + let pid: Int32 + let command: String + let fullCommand: String + let user: String? + } + + struct ReportListener: Identifiable { + let pid: Int32 + let command: String + let fullCommand: String + let user: String? + let expected: Bool + + var id: Int32 { self.pid } + } + + func diagnose(mode: AppState.ConnectionMode) async -> [PortReport] { + if mode == .unconfigured { + return [] + } + let ports = [GatewayEnvironment.gatewayPort()] + var reports: [PortReport] = [] + + for port in ports { + let listeners = await self.listeners(on: port) + let tunnelHealthy = await self.probeGatewayHealthIfNeeded( + port: port, + mode: mode, + listeners: listeners) + reports.append(Self.buildReport( + port: port, + listeners: listeners, + mode: mode, + tunnelHealthy: tunnelHealthy)) + } + + return reports + } + + func probeGatewayHealth(port: Int, timeout: TimeInterval = 2.0) async -> Bool { + let url = URL(string: "http://127.0.0.1:\(port)/")! + let config = URLSessionConfiguration.ephemeral + config.timeoutIntervalForRequest = timeout + config.timeoutIntervalForResource = timeout + let session = URLSession(configuration: config) + var request = URLRequest(url: url) + request.cachePolicy = .reloadIgnoringLocalCacheData + request.timeoutInterval = timeout + do { + let (_, response) = try await session.data(for: request) + return response is HTTPURLResponse + } catch { + return false + } + } + + func isListening(port: Int, pid: Int32? = nil) async -> Bool { + let listeners = await self.listeners(on: port) + if let pid { + return listeners.contains(where: { $0.pid == pid }) + } + return !listeners.isEmpty + } + + private func listeners(on port: Int) async -> [Listener] { + let res = await ShellExecutor.run( + command: ["lsof", "-nP", "-iTCP:\(port)", "-sTCP:LISTEN", "-Fpcn"], + cwd: nil, + env: nil, + timeout: 5) + guard res.ok, let data = res.payload, !data.isEmpty else { return [] } + let text = String(data: data, encoding: .utf8) ?? "" + return Self.parseListeners(from: text) + } + + private static func readFullCommand(pid: Int32) -> String? { + let proc = Process() + proc.executableURL = URL(fileURLWithPath: "/bin/ps") + proc.arguments = ["-p", "\(pid)", "-o", "command="] + let pipe = Pipe() + proc.standardOutput = pipe + proc.standardError = Pipe() + do { + let data = try proc.runAndReadToEnd(from: pipe) + guard !data.isEmpty else { return nil } + return String(data: data, encoding: .utf8)? + .trimmingCharacters(in: .whitespacesAndNewlines) + } catch { + return nil + } + } + + private static func parseListeners(from text: String) -> [Listener] { + var listeners: [Listener] = [] + var currentPid: Int32? + var currentCmd: String? + var currentUser: String? + + func flush() { + if let pid = currentPid, let cmd = currentCmd { + let full = Self.readFullCommand(pid: pid) ?? cmd + listeners.append(Listener(pid: pid, command: cmd, fullCommand: full, user: currentUser)) + } + currentPid = nil + currentCmd = nil + currentUser = nil + } + + for line in text.split(separator: "\n") { + guard let prefix = line.first else { continue } + let value = String(line.dropFirst()) + switch prefix { + case "p": + flush() + currentPid = Int32(value) ?? 0 + case "c": + currentCmd = value + case "u": + currentUser = value + default: + continue + } + } + flush() + return listeners + } + + private static func buildReport( + port: Int, + listeners: [Listener], + mode: AppState.ConnectionMode, + tunnelHealthy: Bool?) -> PortReport + { + let expectedDesc: String + let okPredicate: (Listener) -> Bool + let expectedCommands = ["node", "openclaw", "tsx", "pnpm", "bun"] + + switch mode { + case .remote: + expectedDesc = "SSH tunnel to remote gateway" + okPredicate = { $0.command.lowercased().contains("ssh") } + case .local: + expectedDesc = "Gateway websocket (node/tsx)" + okPredicate = { listener in + let c = listener.command.lowercased() + return expectedCommands.contains { c.contains($0) } + } + case .unconfigured: + expectedDesc = "Gateway not configured" + okPredicate = { _ in false } + } + + if listeners.isEmpty { + let text = "Nothing is listening on \(port) (\(expectedDesc))." + return .init(port: port, expected: expectedDesc, status: .missing(text), listeners: []) + } + + let tunnelUnhealthy = + mode == .remote && port == GatewayEnvironment.gatewayPort() && tunnelHealthy == false + let reportListeners = listeners.map { listener in + var expected = okPredicate(listener) + if tunnelUnhealthy, expected { expected = false } + return ReportListener( + pid: listener.pid, + command: listener.command, + fullCommand: listener.fullCommand, + user: listener.user, + expected: expected) + } + + let offenders = reportListeners.filter { !$0.expected } + if tunnelUnhealthy { + let list = listeners.map { "\($0.command) (\($0.pid))" }.joined(separator: ", ") + let reason = "Port \(port) is served by \(list), but the SSH tunnel is unhealthy." + return .init( + port: port, + expected: expectedDesc, + status: .interference(reason, offenders: offenders), + listeners: reportListeners) + } + if offenders.isEmpty { + let list = listeners.map { "\($0.command) (\($0.pid))" }.joined(separator: ", ") + let okText = "Port \(port) is served by \(list)." + return .init( + port: port, + expected: expectedDesc, + status: .ok(okText), + listeners: reportListeners) + } + + let list = offenders.map { "\($0.command) (\($0.pid))" }.joined(separator: ", ") + let reason = "Port \(port) is held by \(list), expected \(expectedDesc)." + return .init( + port: port, + expected: expectedDesc, + status: .interference(reason, offenders: offenders), + listeners: reportListeners) + } + + private static func executablePath(for pid: Int32) -> String? { + #if canImport(Darwin) + var buffer = [CChar](repeating: 0, count: Int(PATH_MAX)) + let length = proc_pidpath(pid, &buffer, UInt32(buffer.count)) + guard length > 0 else { return nil } + // Drop trailing null and decode as UTF-8. + let trimmed = buffer.prefix { $0 != 0 } + let bytes = trimmed.map { UInt8(bitPattern: $0) } + return String(bytes: bytes, encoding: .utf8) + #else + return nil + #endif + } + + private func kill(_ pid: Int32) async -> Bool { + let term = await ShellExecutor.run(command: ["kill", "-TERM", "\(pid)"], cwd: nil, env: nil, timeout: 2) + if term.ok { return true } + let sigkill = await ShellExecutor.run(command: ["kill", "-KILL", "\(pid)"], cwd: nil, env: nil, timeout: 2) + return sigkill.ok + } + + private func isExpected(_ listener: Listener, port: Int, mode: AppState.ConnectionMode) -> Bool { + let cmd = listener.command.lowercased() + let full = listener.fullCommand.lowercased() + switch mode { + case .remote: + // Remote mode expects an SSH tunnel for the gateway WebSocket port. + if port == GatewayEnvironment.gatewayPort() { return cmd.contains("ssh") } + return false + case .local: + // The gateway daemon may listen as `openclaw` or as its runtime (`node`, `bun`, etc). + if full.contains("gateway-daemon") { return true } + // If args are unavailable, treat a CLI listener as expected. + if cmd.contains("openclaw"), full == cmd { return true } + return false + case .unconfigured: + return false + } + } + + private func probeGatewayHealthIfNeeded( + port: Int, + mode: AppState.ConnectionMode, + listeners: [Listener]) async -> Bool? + { + guard mode == .remote, port == GatewayEnvironment.gatewayPort(), !listeners.isEmpty else { return nil } + let hasSsh = listeners.contains { $0.command.lowercased().contains("ssh") } + guard hasSsh else { return nil } + return await self.probeGatewayHealth(port: port) + } + + private static func loadRecords(from url: URL) -> [Record] { + guard let data = try? Data(contentsOf: url), + let decoded = try? JSONDecoder().decode([Record].self, from: data) + else { return [] } + return decoded + } + + private func save() { + guard let data = try? JSONEncoder().encode(self.records) else { return } + try? data.write(to: Self.recordPath, options: [.atomic]) + } +} + +#if DEBUG +extension PortGuardian { + static func _testParseListeners(_ text: String) -> [( + pid: Int32, + command: String, + fullCommand: String, + user: String?)] + { + self.parseListeners(from: text).map { ($0.pid, $0.command, $0.fullCommand, $0.user) } + } + + static func _testBuildReport( + port: Int, + mode: AppState.ConnectionMode, + listeners: [(pid: Int32, command: String, fullCommand: String, user: String?)]) -> PortReport + { + let mapped = listeners.map { Listener( + pid: $0.pid, + command: $0.command, + fullCommand: $0.fullCommand, + user: $0.user) } + return Self.buildReport(port: port, listeners: mapped, mode: mode, tunnelHealthy: nil) + } +} +#endif diff --git a/apps/macos/Sources/OpenClaw/PresenceReporter.swift b/apps/macos/Sources/OpenClaw/PresenceReporter.swift new file mode 100644 index 0000000000000000000000000000000000000000..16d70b8a92c0c95d1943cf497e6838b923179539 --- /dev/null +++ b/apps/macos/Sources/OpenClaw/PresenceReporter.swift @@ -0,0 +1,158 @@ +import Cocoa +import Darwin +import Foundation +import OSLog + +@MainActor +final class PresenceReporter { + static let shared = PresenceReporter() + + private let logger = Logger(subsystem: "ai.openclaw", category: "presence") + private var task: Task? + private let interval: TimeInterval = 180 // a few minutes + private let instanceId: String = InstanceIdentity.instanceId + + func start() { + guard self.task == nil else { return } + self.task = Task.detached { [weak self] in + guard let self else { return } + await self.push(reason: "launch") + while !Task.isCancelled { + try? await Task.sleep(nanoseconds: UInt64(self.interval * 1_000_000_000)) + await self.push(reason: "periodic") + } + } + } + + func stop() { + self.task?.cancel() + self.task = nil + } + + @Sendable + private func push(reason: String) async { + let mode = await MainActor.run { AppStateStore.shared.connectionMode.rawValue } + let host = InstanceIdentity.displayName + let ip = Self.primaryIPv4Address() ?? "ip-unknown" + let version = Self.appVersionString() + let platform = Self.platformString() + let lastInput = Self.lastInputSeconds() + let text = Self.composePresenceSummary(mode: mode, reason: reason) + var params: [String: AnyHashable] = [ + "instanceId": AnyHashable(self.instanceId), + "host": AnyHashable(host), + "ip": AnyHashable(ip), + "mode": AnyHashable(mode), + "version": AnyHashable(version), + "platform": AnyHashable(platform), + "deviceFamily": AnyHashable("Mac"), + "reason": AnyHashable(reason), + ] + if let model = InstanceIdentity.modelIdentifier { params["modelIdentifier"] = AnyHashable(model) } + if let lastInput { params["lastInputSeconds"] = AnyHashable(lastInput) } + do { + try await ControlChannel.shared.sendSystemEvent(text, params: params) + } catch { + self.logger.error("presence send failed: \(error.localizedDescription, privacy: .public)") + } + } + + /// Fire an immediate presence beacon (e.g., right after connecting). + func sendImmediate(reason: String = "connect") { + Task { await self.push(reason: reason) } + } + + private static func composePresenceSummary(mode: String, reason: String) -> String { + let host = InstanceIdentity.displayName + let ip = Self.primaryIPv4Address() ?? "ip-unknown" + let version = Self.appVersionString() + let lastInput = Self.lastInputSeconds() + let lastLabel = lastInput.map { "last input \($0)s ago" } ?? "last input unknown" + return "Node: \(host) (\(ip)) · app \(version) · \(lastLabel) · mode \(mode) · reason \(reason)" + } + + private static func appVersionString() -> String { + let version = Bundle.main.object(forInfoDictionaryKey: "CFBundleShortVersionString") as? String ?? "dev" + if let build = Bundle.main.object(forInfoDictionaryKey: "CFBundleVersion") as? String { + let trimmed = build.trimmingCharacters(in: .whitespacesAndNewlines) + if !trimmed.isEmpty, trimmed != version { + return "\(version) (\(trimmed))" + } + } + return version + } + + private static func platformString() -> String { + let v = ProcessInfo.processInfo.operatingSystemVersion + return "macos \(v.majorVersion).\(v.minorVersion).\(v.patchVersion)" + } + + private static func lastInputSeconds() -> Int? { + let anyEvent = CGEventType(rawValue: UInt32.max) ?? .null + let seconds = CGEventSource.secondsSinceLastEventType(.combinedSessionState, eventType: anyEvent) + if seconds.isNaN || seconds.isInfinite || seconds < 0 { return nil } + return Int(seconds.rounded()) + } + + private static func primaryIPv4Address() -> String? { + var addrList: UnsafeMutablePointer? + guard getifaddrs(&addrList) == 0, let first = addrList else { return nil } + defer { freeifaddrs(addrList) } + + var fallback: String? + var en0: String? + + for ptr in sequence(first: first, next: { $0.pointee.ifa_next }) { + let flags = Int32(ptr.pointee.ifa_flags) + let isUp = (flags & IFF_UP) != 0 + let isLoopback = (flags & IFF_LOOPBACK) != 0 + let name = String(cString: ptr.pointee.ifa_name) + let family = ptr.pointee.ifa_addr.pointee.sa_family + if !isUp || isLoopback || family != UInt8(AF_INET) { continue } + + var addr = ptr.pointee.ifa_addr.pointee + var buffer = [CChar](repeating: 0, count: Int(NI_MAXHOST)) + let result = getnameinfo( + &addr, + socklen_t(ptr.pointee.ifa_addr.pointee.sa_len), + &buffer, + socklen_t(buffer.count), + nil, + 0, + NI_NUMERICHOST) + guard result == 0 else { continue } + let len = buffer.prefix { $0 != 0 } + let bytes = len.map { UInt8(bitPattern: $0) } + guard let ip = String(bytes: bytes, encoding: .utf8) else { continue } + + if name == "en0" { en0 = ip; break } + if fallback == nil { fallback = ip } + } + + return en0 ?? fallback + } +} + +#if DEBUG +extension PresenceReporter { + static func _testComposePresenceSummary(mode: String, reason: String) -> String { + self.composePresenceSummary(mode: mode, reason: reason) + } + + static func _testAppVersionString() -> String { + self.appVersionString() + } + + static func _testPlatformString() -> String { + self.platformString() + } + + static func _testLastInputSeconds() -> Int? { + self.lastInputSeconds() + } + + static func _testPrimaryIPv4Address() -> String? { + self.primaryIPv4Address() + } +} +#endif diff --git a/apps/macos/Sources/OpenClaw/Process+PipeRead.swift b/apps/macos/Sources/OpenClaw/Process+PipeRead.swift new file mode 100644 index 0000000000000000000000000000000000000000..7c0f7fe0ca3df33e30c739e1c5bfa51ddbb3f634 --- /dev/null +++ b/apps/macos/Sources/OpenClaw/Process+PipeRead.swift @@ -0,0 +1,11 @@ +import Foundation + +extension Process { + /// Runs the process and drains the given pipe before waiting to avoid blocking on full buffers. + func runAndReadToEnd(from pipe: Pipe) throws -> Data { + try self.run() + let data = pipe.fileHandleForReading.readToEndSafely() + self.waitUntilExit() + return data + } +} diff --git a/apps/macos/Sources/OpenClaw/ProcessInfo+OpenClaw.swift b/apps/macos/Sources/OpenClaw/ProcessInfo+OpenClaw.swift new file mode 100644 index 0000000000000000000000000000000000000000..65ea48e0f2d018bdd625c451d851e57633168386 --- /dev/null +++ b/apps/macos/Sources/OpenClaw/ProcessInfo+OpenClaw.swift @@ -0,0 +1,25 @@ +import Foundation + +extension ProcessInfo { + var isPreview: Bool { + guard let raw = getenv("XCODE_RUNNING_FOR_PREVIEWS") else { return false } + return String(cString: raw) == "1" + } + + var isNixMode: Bool { + if let raw = getenv("OPENCLAW_NIX_MODE"), String(cString: raw) == "1" { return true } + return UserDefaults.standard.bool(forKey: "openclaw.nixMode") + } + + var isRunningTests: Bool { + // SwiftPM tests load one or more `.xctest` bundles. With Swift Testing, `Bundle.main` is not + // guaranteed to be the `.xctest` bundle, so check all loaded bundles. + if Bundle.allBundles.contains(where: { $0.bundleURL.pathExtension == "xctest" }) { return true } + if Bundle.main.bundleURL.pathExtension == "xctest" { return true } + + // Backwards-compatible fallbacks for runners that still set XCTest env vars. + return self.environment["XCTestConfigurationFilePath"] != nil + || self.environment["XCTestBundlePath"] != nil + || self.environment["XCTestSessionIdentifier"] != nil + } +} diff --git a/apps/macos/Sources/OpenClaw/RemotePortTunnel.swift b/apps/macos/Sources/OpenClaw/RemotePortTunnel.swift new file mode 100644 index 0000000000000000000000000000000000000000..6502d2ad91602ed422cb01ed960b047f9da42c40 --- /dev/null +++ b/apps/macos/Sources/OpenClaw/RemotePortTunnel.swift @@ -0,0 +1,317 @@ +import Foundation +import Network +import OSLog +#if canImport(Darwin) +import Darwin +#endif + +/// Port forwarding tunnel for remote mode. +/// +/// Uses `ssh -N -L` to forward the remote gateway ports to localhost. +final class RemotePortTunnel { + private static let logger = Logger(subsystem: "ai.openclaw", category: "remote.tunnel") + + let process: Process + let localPort: UInt16? + private let stderrHandle: FileHandle? + + private init(process: Process, localPort: UInt16?, stderrHandle: FileHandle?) { + self.process = process + self.localPort = localPort + self.stderrHandle = stderrHandle + } + + deinit { + Self.cleanupStderr(self.stderrHandle) + let pid = self.process.processIdentifier + self.process.terminate() + Task { await PortGuardian.shared.removeRecord(pid: pid) } + } + + func terminate() { + Self.cleanupStderr(self.stderrHandle) + let pid = self.process.processIdentifier + if self.process.isRunning { + self.process.terminate() + self.process.waitUntilExit() + } + Task { await PortGuardian.shared.removeRecord(pid: pid) } + } + + static func create( + remotePort: Int, + preferredLocalPort: UInt16? = nil, + allowRemoteUrlOverride: Bool = true, + allowRandomLocalPort: Bool = true) async throws -> RemotePortTunnel + { + let settings = CommandResolver.connectionSettings() + guard settings.mode == .remote, let parsed = CommandResolver.parseSSHTarget(settings.target) else { + throw NSError( + domain: "RemotePortTunnel", + code: 3, + userInfo: [NSLocalizedDescriptionKey: "Remote mode is not configured"]) + } + + let localPort = try await Self.findPort( + preferred: preferredLocalPort, + allowRandom: allowRandomLocalPort) + let sshHost = parsed.host.trimmingCharacters(in: .whitespacesAndNewlines) + let remotePortOverride = + allowRemoteUrlOverride && remotePort == GatewayEnvironment.gatewayPort() + ? Self.resolveRemotePortOverride(for: sshHost) + : nil + let resolvedRemotePort = remotePortOverride ?? remotePort + if let override = remotePortOverride { + Self.logger.info( + "ssh tunnel remote port override " + + "host=\(sshHost, privacy: .public) port=\(override, privacy: .public)") + } else { + Self.logger.debug( + "ssh tunnel using default remote port " + + "host=\(sshHost, privacy: .public) port=\(remotePort, privacy: .public)") + } + let options: [String] = [ + "-o", "BatchMode=yes", + "-o", "ExitOnForwardFailure=yes", + "-o", "StrictHostKeyChecking=accept-new", + "-o", "UpdateHostKeys=yes", + "-o", "ServerAliveInterval=15", + "-o", "ServerAliveCountMax=3", + "-o", "TCPKeepAlive=yes", + "-N", + "-L", "\(localPort):127.0.0.1:\(resolvedRemotePort)", + ] + let identity = settings.identity.trimmingCharacters(in: .whitespacesAndNewlines) + let args = CommandResolver.sshArguments( + target: parsed, + identity: identity, + options: options) + + let process = Process() + process.executableURL = URL(fileURLWithPath: "/usr/bin/ssh") + process.arguments = args + + let pipe = Pipe() + process.standardError = pipe + let stderrHandle = pipe.fileHandleForReading + + // Consume stderr so ssh cannot block if it logs. + stderrHandle.readabilityHandler = { handle in + let data = handle.readSafely(upToCount: 64 * 1024) + guard !data.isEmpty else { + // EOF (or read failure): stop monitoring to avoid spinning on a closed pipe. + Self.cleanupStderr(handle) + return + } + guard let line = String(data: data, encoding: .utf8)? + .trimmingCharacters(in: .whitespacesAndNewlines), + !line.isEmpty + else { return } + Self.logger.error("ssh tunnel stderr: \(line, privacy: .public)") + } + process.terminationHandler = { _ in + Self.cleanupStderr(stderrHandle) + } + + try process.run() + + // If ssh exits immediately (e.g. local port already in use), surface stderr and ensure we stop monitoring. + try? await Task.sleep(nanoseconds: 150_000_000) // 150ms + if !process.isRunning { + let stderr = Self.drainStderr(stderrHandle) + let msg = stderr.isEmpty ? "ssh tunnel exited immediately" : "ssh tunnel failed: \(stderr)" + throw NSError(domain: "RemotePortTunnel", code: 4, userInfo: [NSLocalizedDescriptionKey: msg]) + } + + // Track tunnel so we can clean up stale listeners on restart. + Task { + await PortGuardian.shared.record( + port: Int(localPort), + pid: process.processIdentifier, + command: process.executableURL?.path ?? "ssh", + mode: CommandResolver.connectionSettings().mode) + } + + return RemotePortTunnel(process: process, localPort: localPort, stderrHandle: stderrHandle) + } + + private static func resolveRemotePortOverride(for sshHost: String) -> Int? { + let root = OpenClawConfigFile.loadDict() + guard let gateway = root["gateway"] as? [String: Any], + let remote = gateway["remote"] as? [String: Any], + let urlRaw = remote["url"] as? String + else { + return nil + } + let trimmed = urlRaw.trimmingCharacters(in: .whitespacesAndNewlines) + guard !trimmed.isEmpty, let url = URL(string: trimmed), let port = url.port else { + return nil + } + guard let host = url.host?.trimmingCharacters(in: .whitespacesAndNewlines), + !host.isEmpty + else { + return nil + } + let sshKey = Self.hostKey(sshHost) + let urlKey = Self.hostKey(host) + guard !sshKey.isEmpty, !urlKey.isEmpty else { return nil } + guard sshKey == urlKey else { + Self.logger.debug( + "remote url host mismatch sshHost=\(sshHost, privacy: .public) urlHost=\(host, privacy: .public)") + return nil + } + return port + } + + private static func hostKey(_ host: String) -> String { + let trimmed = host.trimmingCharacters(in: .whitespacesAndNewlines).lowercased() + guard !trimmed.isEmpty else { return "" } + if trimmed.contains(":") { return trimmed } + let digits = CharacterSet(charactersIn: "0123456789.") + if trimmed.rangeOfCharacter(from: digits.inverted) == nil { + return trimmed + } + return trimmed.split(separator: ".").first.map(String.init) ?? trimmed + } + + private static func findPort(preferred: UInt16?, allowRandom: Bool) async throws -> UInt16 { + if let preferred, self.portIsFree(preferred) { return preferred } + if let preferred, !allowRandom { + throw NSError( + domain: "RemotePortTunnel", + code: 5, + userInfo: [ + NSLocalizedDescriptionKey: "Local port \(preferred) is unavailable", + ]) + } + + return try await withCheckedThrowingContinuation { cont in + let queue = DispatchQueue(label: "ai.openclaw.remote.tunnel.port", qos: .utility) + do { + let listener = try NWListener(using: .tcp, on: .any) + listener.newConnectionHandler = { connection in connection.cancel() } + listener.stateUpdateHandler = { state in + switch state { + case .ready: + if let port = listener.port?.rawValue { + listener.stateUpdateHandler = nil + listener.cancel() + cont.resume(returning: port) + } + case let .failed(error): + listener.stateUpdateHandler = nil + listener.cancel() + cont.resume(throwing: error) + default: + break + } + } + listener.start(queue: queue) + } catch { + cont.resume(throwing: error) + } + } + } + + private static func portIsFree(_ port: UInt16) -> Bool { + #if canImport(Darwin) + // NWListener can succeed even when only one address family is held. Mirror what ssh needs by checking + // both 127.0.0.1 and ::1 for availability. + return self.canBindIPv4(port) && self.canBindIPv6(port) + #else + do { + let listener = try NWListener(using: .tcp, on: NWEndpoint.Port(rawValue: port)!) + listener.cancel() + return true + } catch { + return false + } + #endif + } + + #if canImport(Darwin) + private static func canBindIPv4(_ port: UInt16) -> Bool { + let fd = socket(AF_INET, SOCK_STREAM, 0) + guard fd >= 0 else { return false } + defer { _ = Darwin.close(fd) } + + var one: Int32 = 1 + _ = setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &one, socklen_t(MemoryLayout.size(ofValue: one))) + + var addr = sockaddr_in() + addr.sin_len = UInt8(MemoryLayout.size) + addr.sin_family = sa_family_t(AF_INET) + addr.sin_port = port.bigEndian + addr.sin_addr = in_addr(s_addr: inet_addr("127.0.0.1")) + + let result = withUnsafePointer(to: &addr) { ptr in + ptr.withMemoryRebound(to: sockaddr.self, capacity: 1) { sa in + Darwin.bind(fd, sa, socklen_t(MemoryLayout.size)) + } + } + return result == 0 + } + + private static func canBindIPv6(_ port: UInt16) -> Bool { + let fd = socket(AF_INET6, SOCK_STREAM, 0) + guard fd >= 0 else { return false } + defer { _ = Darwin.close(fd) } + + var one: Int32 = 1 + _ = setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &one, socklen_t(MemoryLayout.size(ofValue: one))) + + var addr = sockaddr_in6() + addr.sin6_len = UInt8(MemoryLayout.size) + addr.sin6_family = sa_family_t(AF_INET6) + addr.sin6_port = port.bigEndian + var loopback = in6_addr() + _ = withUnsafeMutablePointer(to: &loopback) { ptr in + inet_pton(AF_INET6, "::1", ptr) + } + addr.sin6_addr = loopback + + let result = withUnsafePointer(to: &addr) { ptr in + ptr.withMemoryRebound(to: sockaddr.self, capacity: 1) { sa in + Darwin.bind(fd, sa, socklen_t(MemoryLayout.size)) + } + } + return result == 0 + } + #endif + + private static func cleanupStderr(_ handle: FileHandle?) { + guard let handle else { return } + Self.cleanupStderr(handle) + } + + private static func cleanupStderr(_ handle: FileHandle) { + if handle.readabilityHandler != nil { + handle.readabilityHandler = nil + } + try? handle.close() + } + + private static func drainStderr(_ handle: FileHandle) -> String { + handle.readabilityHandler = nil + defer { try? handle.close() } + + do { + let data = try handle.readToEnd() ?? Data() + return String(data: data, encoding: .utf8)? + .trimmingCharacters(in: .whitespacesAndNewlines) ?? "" + } catch { + self.logger.debug("Failed to drain ssh stderr: \(error, privacy: .public)") + return "" + } + } + + #if SWIFT_PACKAGE + static func _testPortIsFree(_ port: UInt16) -> Bool { + self.portIsFree(port) + } + + static func _testDrainStderr(_ handle: FileHandle) -> String { + self.drainStderr(handle) + } + #endif +} diff --git a/apps/macos/Sources/OpenClaw/RemoteTunnelManager.swift b/apps/macos/Sources/OpenClaw/RemoteTunnelManager.swift new file mode 100644 index 0000000000000000000000000000000000000000..e8f0da6f09145a849bbfb22ca18519ad4706da86 --- /dev/null +++ b/apps/macos/Sources/OpenClaw/RemoteTunnelManager.swift @@ -0,0 +1,122 @@ +import Foundation +import OSLog + +/// Manages the SSH tunnel that forwards the remote gateway/control port to localhost. +actor RemoteTunnelManager { + static let shared = RemoteTunnelManager() + + private let logger = Logger(subsystem: "ai.openclaw", category: "remote-tunnel") + private var controlTunnel: RemotePortTunnel? + private var restartInFlight = false + private var lastRestartAt: Date? + private let restartBackoffSeconds: TimeInterval = 2.0 + + func controlTunnelPortIfRunning() async -> UInt16? { + if self.restartInFlight { + self.logger.info("control tunnel restart in flight; skipping reuse check") + return nil + } + if let tunnel = self.controlTunnel, + tunnel.process.isRunning, + let local = tunnel.localPort + { + let pid = tunnel.process.processIdentifier + if await PortGuardian.shared.isListening(port: Int(local), pid: pid) { + self.logger.info("reusing active SSH tunnel localPort=\(local, privacy: .public)") + return local + } + self.logger.error( + "active SSH tunnel on port \(local, privacy: .public) is not listening; restarting") + await self.beginRestart() + tunnel.terminate() + self.controlTunnel = nil + } + // If a previous OpenClaw run already has an SSH listener on the expected port (common after restarts), + // reuse it instead of spawning new ssh processes that immediately fail with "Address already in use". + let desiredPort = UInt16(GatewayEnvironment.gatewayPort()) + if let desc = await PortGuardian.shared.describe(port: Int(desiredPort)), + self.isSshProcess(desc) + { + self.logger.info( + "reusing existing SSH tunnel listener " + + "localPort=\(desiredPort, privacy: .public) " + + "pid=\(desc.pid, privacy: .public)") + return desiredPort + } + return nil + } + + /// Ensure an SSH tunnel is running for the gateway control port. + /// Returns the local forwarded port (usually the configured gateway port). + func ensureControlTunnel() async throws -> UInt16 { + let settings = CommandResolver.connectionSettings() + guard settings.mode == .remote else { + throw NSError( + domain: "RemoteTunnel", + code: 1, + userInfo: [NSLocalizedDescriptionKey: "Remote mode is not enabled"]) + } + + let identitySet = !settings.identity.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty + self.logger.info( + "ensure SSH tunnel target=\(settings.target, privacy: .public) " + + "identitySet=\(identitySet, privacy: .public)") + + if let local = await self.controlTunnelPortIfRunning() { return local } + await self.waitForRestartBackoffIfNeeded() + + let desiredPort = UInt16(GatewayEnvironment.gatewayPort()) + let tunnel = try await RemotePortTunnel.create( + remotePort: GatewayEnvironment.gatewayPort(), + preferredLocalPort: desiredPort, + allowRandomLocalPort: false) + self.controlTunnel = tunnel + self.endRestart() + let resolvedPort = tunnel.localPort ?? desiredPort + self.logger.info("ssh tunnel ready localPort=\(resolvedPort, privacy: .public)") + return tunnel.localPort ?? desiredPort + } + + func stopAll() { + self.controlTunnel?.terminate() + self.controlTunnel = nil + } + + private func isSshProcess(_ desc: PortGuardian.Descriptor) -> Bool { + let cmd = desc.command.lowercased() + if cmd.contains("ssh") { return true } + if let path = desc.executablePath?.lowercased(), path.contains("/ssh") { return true } + return false + } + + private func beginRestart() async { + guard !self.restartInFlight else { return } + self.restartInFlight = true + self.lastRestartAt = Date() + self.logger.info("control tunnel restart started") + Task { [weak self] in + guard let self else { return } + try? await Task.sleep(nanoseconds: UInt64(self.restartBackoffSeconds * 1_000_000_000)) + await self.endRestart() + } + } + + private func endRestart() { + if self.restartInFlight { + self.restartInFlight = false + self.logger.info("control tunnel restart finished") + } + } + + private func waitForRestartBackoffIfNeeded() async { + guard let last = self.lastRestartAt else { return } + let elapsed = Date().timeIntervalSince(last) + let remaining = self.restartBackoffSeconds - elapsed + guard remaining > 0 else { return } + self.logger.info( + "control tunnel restart backoff \(remaining, privacy: .public)s") + try? await Task.sleep(nanoseconds: UInt64(remaining * 1_000_000_000)) + } + + // Keep tunnel reuse lightweight; restart only when the listener disappears. +} diff --git a/apps/macos/Sources/OpenClaw/Resources/DeviceModels/LICENSE.apple-device-identifiers.txt b/apps/macos/Sources/OpenClaw/Resources/DeviceModels/LICENSE.apple-device-identifiers.txt new file mode 100644 index 0000000000000000000000000000000000000000..d1b9e4b3ce5b02d48933a2e46b4edb08e7c791ed --- /dev/null +++ b/apps/macos/Sources/OpenClaw/Resources/DeviceModels/LICENSE.apple-device-identifiers.txt @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2021 Kyle Seongwoo Jun + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/apps/macos/Sources/OpenClaw/Resources/DeviceModels/NOTICE.md b/apps/macos/Sources/OpenClaw/Resources/DeviceModels/NOTICE.md new file mode 100644 index 0000000000000000000000000000000000000000..664e78d7bc987f6a5f3090448db18031995980b5 --- /dev/null +++ b/apps/macos/Sources/OpenClaw/Resources/DeviceModels/NOTICE.md @@ -0,0 +1,9 @@ +# Apple device identifier mappings + +This directory includes model identifier → human-readable name mappings derived from the open-source project: + +- `kyle-seongwoo-jun/apple-device-identifiers` + - iOS mapping pinned to commit `8e7388b29da046183f5d976eb74dbb2f2acda955` + - macOS mapping pinned to commit `98ca75324f7a88c1649eb5edfc266ef47b7b8193` + +See `LICENSE.apple-device-identifiers.txt` for license terms. diff --git a/apps/macos/Sources/OpenClaw/Resources/DeviceModels/ios-device-identifiers.json b/apps/macos/Sources/OpenClaw/Resources/DeviceModels/ios-device-identifiers.json new file mode 100644 index 0000000000000000000000000000000000000000..76caa5452ea20b7069c30353d0ac2f83cea77184 --- /dev/null +++ b/apps/macos/Sources/OpenClaw/Resources/DeviceModels/ios-device-identifiers.json @@ -0,0 +1,176 @@ +{ + "i386": "iPhone Simulator", + "x86_64": "iPhone Simulator", + "arm64": "iPhone Simulator", + "iPhone1,1": "iPhone", + "iPhone1,2": "iPhone 3G", + "iPhone2,1": "iPhone 3GS", + "iPhone3,1": "iPhone 4", + "iPhone3,2": "iPhone 4", + "iPhone3,3": "iPhone 4", + "iPhone4,1": "iPhone 4s", + "iPhone5,1": "iPhone 5", + "iPhone5,2": "iPhone 5", + "iPhone5,3": "iPhone 5c", + "iPhone5,4": "iPhone 5c", + "iPhone6,1": "iPhone 5s", + "iPhone6,2": "iPhone 5s", + "iPhone7,1": "iPhone 6 Plus", + "iPhone7,2": "iPhone 6", + "iPhone8,1": "iPhone 6s", + "iPhone8,2": "iPhone 6s Plus", + "iPhone8,4": "iPhone SE (1st generation)", + "iPhone9,1": "iPhone 7", + "iPhone9,2": "iPhone 7 Plus", + "iPhone9,3": "iPhone 7", + "iPhone9,4": "iPhone 7 Plus", + "iPhone10,1": "iPhone 8", + "iPhone10,2": "iPhone 8 Plus", + "iPhone10,3": "iPhone X", + "iPhone10,4": "iPhone 8", + "iPhone10,5": "iPhone 8 Plus", + "iPhone10,6": "iPhone X", + "iPhone11,2": "iPhone XS", + "iPhone11,4": "iPhone XS Max", + "iPhone11,6": "iPhone XS Max", + "iPhone11,8": "iPhone XR", + "iPhone12,1": "iPhone 11", + "iPhone12,3": "iPhone 11 Pro", + "iPhone12,5": "iPhone 11 Pro Max", + "iPhone12,8": "iPhone SE (2nd generation)", + "iPhone13,1": "iPhone 12 mini", + "iPhone13,2": "iPhone 12", + "iPhone13,3": "iPhone 12 Pro", + "iPhone13,4": "iPhone 12 Pro Max", + "iPhone14,2": "iPhone 13 Pro", + "iPhone14,3": "iPhone 13 Pro Max", + "iPhone14,4": "iPhone 13 mini", + "iPhone14,5": "iPhone 13", + "iPhone14,6": "iPhone SE (3rd generation)", + "iPhone14,7": "iPhone 14", + "iPhone14,8": "iPhone 14 Plus", + "iPhone15,2": "iPhone 14 Pro", + "iPhone15,3": "iPhone 14 Pro Max", + "iPhone15,4": "iPhone 15", + "iPhone15,5": "iPhone 15 Plus", + "iPhone16,1": "iPhone 15 Pro", + "iPhone16,2": "iPhone 15 Pro Max", + "iPhone17,1": "iPhone 16 Pro", + "iPhone17,2": "iPhone 16 Pro Max", + "iPhone17,3": "iPhone 16", + "iPhone17,4": "iPhone 16 Plus", + "iPhone17,5": "iPhone 16e", + "iPhone18,1": "iPhone 17 Pro", + "iPhone18,2": "iPhone 17 Pro Max", + "iPhone18,3": "iPhone 17", + "iPhone18,4": "iPhone Air", + "iPad1,1": "iPad", + "iPad1,2": "iPad", + "iPad2,1": "iPad 2", + "iPad2,2": "iPad 2", + "iPad2,3": "iPad 2", + "iPad2,4": "iPad 2", + "iPad2,5": "iPad mini", + "iPad2,6": "iPad mini", + "iPad2,7": "iPad mini", + "iPad3,1": "iPad (3rd generation)", + "iPad3,2": "iPad (3rd generation)", + "iPad3,3": "iPad (3rd generation)", + "iPad3,4": "iPad (4th generation)", + "iPad3,5": "iPad (4th generation)", + "iPad3,6": "iPad (4th generation)", + "iPad4,1": "iPad Air", + "iPad4,2": "iPad Air", + "iPad4,3": "iPad Air", + "iPad4,4": "iPad mini 2", + "iPad4,5": "iPad mini 2", + "iPad4,6": "iPad mini 2", + "iPad4,7": "iPad mini 3", + "iPad4,8": "iPad mini 3", + "iPad4,9": "iPad mini 3", + "iPad5,1": "iPad mini 4", + "iPad5,2": "iPad mini 4", + "iPad5,3": "iPad Air 2", + "iPad5,4": "iPad Air 2", + "iPad6,3": "iPad Pro (9.7-inch)", + "iPad6,4": "iPad Pro (9.7-inch)", + "iPad6,7": "iPad Pro (12.9-inch)", + "iPad6,8": "iPad Pro (12.9-inch)", + "iPad6,11": "iPad (5th generation)", + "iPad6,12": "iPad (5th generation)", + "iPad7,1": "iPad Pro (12.9-inch) (2nd generation)", + "iPad7,2": "iPad Pro (12.9-inch) (2nd generation)", + "iPad7,3": "iPad Pro (10.5-inch)", + "iPad7,4": "iPad Pro (10.5-inch)", + "iPad7,5": "iPad (6th generation)", + "iPad7,6": "iPad (6th generation)", + "iPad7,11": "iPad (7th generation)", + "iPad7,12": "iPad (7th generation)", + "iPad8,1": "iPad Pro (11-inch)", + "iPad8,2": "iPad Pro (11-inch)", + "iPad8,3": "iPad Pro (11-inch)", + "iPad8,4": "iPad Pro (11-inch)", + "iPad8,5": "iPad Pro (12.9-inch) (3rd generation)", + "iPad8,6": "iPad Pro (12.9-inch) (3rd generation)", + "iPad8,7": "iPad Pro (12.9-inch) (3rd generation)", + "iPad8,8": "iPad Pro (12.9-inch) (3rd generation)", + "iPad8,9": "iPad Pro (11-inch) (2nd generation)", + "iPad8,10": "iPad Pro (11-inch) (2nd generation)", + "iPad8,11": "iPad Pro (12.9-inch) (4th generation)", + "iPad8,12": "iPad Pro (12.9-inch) (4th generation)", + "iPad11,1": "iPad mini (5th generation)", + "iPad11,2": "iPad mini (5th generation)", + "iPad11,3": "iPad Air (3rd generation)", + "iPad11,4": "iPad Air (3rd generation)", + "iPad11,6": "iPad (8th generation)", + "iPad11,7": "iPad (8th generation)", + "iPad12,1": "iPad (9th generation)", + "iPad12,2": "iPad (9th generation)", + "iPad13,1": "iPad Air (4th generation)", + "iPad13,2": "iPad Air (4th generation)", + "iPad13,4": "iPad Pro (11-inch) (3rd generation)", + "iPad13,5": "iPad Pro (11-inch) (3rd generation)", + "iPad13,6": "iPad Pro (11-inch) (3rd generation)", + "iPad13,7": "iPad Pro (11-inch) (3rd generation)", + "iPad13,8": "iPad Pro (12.9-inch) (5th generation)", + "iPad13,9": "iPad Pro (12.9-inch) (5th generation)", + "iPad13,10": "iPad Pro (12.9-inch) (5th generation)", + "iPad13,11": "iPad Pro (12.9-inch) (5th generation)", + "iPad13,16": "iPad Air (5th generation)", + "iPad13,17": "iPad Air (5th generation)", + "iPad13,18": "iPad (10th generation)", + "iPad13,19": "iPad (10th generation)", + "iPad14,1": "iPad mini (6th generation)", + "iPad14,2": "iPad mini (6th generation)", + "iPad14,3": "iPad Pro (11-inch) (4th generation)", + "iPad14,4": "iPad Pro (11-inch) (4th generation)", + "iPad14,5": "iPad Pro (12.9-inch) (6th generation)", + "iPad14,6": "iPad Pro (12.9-inch) (6th generation)", + "iPad14,8": "iPad Air 11-inch (M2)", + "iPad14,9": "iPad Air 11-inch (M2)", + "iPad14,10": "iPad Air 13-inch (M2)", + "iPad14,11": "iPad Air 13-inch (M2)", + "iPad15,3": "iPad Air 11-inch (M3)", + "iPad15,4": "iPad Air 11-inch (M3)", + "iPad15,5": "iPad Air 13-inch (M3)", + "iPad15,6": "iPad Air 13-inch (M3)", + "iPad15,7": "iPad (A16)", + "iPad15,8": "iPad (A16)", + "iPad16,1": "iPad mini (A17 Pro)", + "iPad16,2": "iPad mini (A17 Pro)", + "iPad16,3": "iPad Pro 11-inch (M4)", + "iPad16,4": "iPad Pro 11-inch (M4)", + "iPad16,5": "iPad Pro 13-inch (M4)", + "iPad16,6": "iPad Pro 13-inch (M4)", + "iPad17,1": "iPad Pro 11-inch (M5)", + "iPad17,2": "iPad Pro 11-inch (M5)", + "iPad17,3": "iPad Pro 13-inch (M5)", + "iPad17,4": "iPad Pro 13-inch (M5)", + "iPod1,1": "iPod touch", + "iPod2,1": "iPod touch (2nd generation)", + "iPod3,1": "iPod touch (3rd generation)", + "iPod4,1": "iPod touch (4th generation)", + "iPod5,1": "iPod touch (5th generation)", + "iPod7,1": "iPod touch (6th generation)", + "iPod9,1": "iPod touch (7th generation)" +} diff --git a/apps/macos/Sources/OpenClaw/Resources/DeviceModels/mac-device-identifiers.json b/apps/macos/Sources/OpenClaw/Resources/DeviceModels/mac-device-identifiers.json new file mode 100644 index 0000000000000000000000000000000000000000..03d5a5eccb16cdbdd1784ac906ea1f31be9b303b --- /dev/null +++ b/apps/macos/Sources/OpenClaw/Resources/DeviceModels/mac-device-identifiers.json @@ -0,0 +1,214 @@ +{ + "iMac9,1": [ + "iMac (20-inch, Early 2009)", + "iMac (24-inch, Early 2009)" + ], + "iMac10,1": [ + "iMac (21.5-inch, Late 2009)", + "iMac (27-inch, Late 2009)" + ], + "iMac11,2": "iMac (21.5-inch, Mid 2010)", + "iMac11,3": "iMac (27-inch, Mid 2010)", + "iMac12,1": "iMac (21.5-inch, Mid 2011)", + "iMac12,2": "iMac (27-inch, Mid 2011)", + "iMac13,1": "iMac (21.5-inch, Late 2012)", + "iMac13,2": "iMac (27-inch, Late 2012)", + "iMac14,1": "iMac (21.5-inch, Late 2013)", + "iMac14,2": "iMac (27-inch, Late 2013)", + "iMac14,4": "iMac (21.5-inch, Mid 2014)", + "iMac15,1": [ + "iMac (Retina 5K, 27-inch, Late 2014)", + "iMac (Retina 5K, 27-inch, Mid 2015)" + ], + "iMac16,1": "iMac (21.5-inch, Late 2015)", + "iMac16,2": "iMac (Retina 4K, 21.5-inch, Late 2015)", + "iMac17,1": "iMac (Retina 5K, 27-inch, Late 2015)", + "iMac18,1": "iMac (21.5-inch, 2017)", + "iMac18,2": "iMac (Retina 4K, 21.5-inch, 2017)", + "iMac18,3": "iMac (Retina 5K, 27-inch, 2017)", + "iMac19,1": "iMac (Retina 5K, 27-inch, 2019)", + "iMac19,2": "iMac (Retina 4K, 21.5-inch, 2019)", + "iMac20,1": "iMac (Retina 5K, 27-inch, 2020)", + "iMac20,2": "iMac (Retina 5K, 27-inch, 2020)", + "iMac21,1": "iMac (24-inch, M1, 2021)", + "iMac21,2": "iMac (24-inch, M1, 2021)", + "iMacPro1,1": "iMac Pro (2017)", + "Mac13,1": "Mac Studio (2022)", + "Mac13,2": "Mac Studio (2022)", + "Mac14,2": "MacBook Air (M2, 2022)", + "Mac14,3": "Mac mini (2023)", + "Mac14,5": "MacBook Pro (14-inch, 2023)", + "Mac14,6": "MacBook Pro (16-inch, 2023)", + "Mac14,7": "MacBook Pro (13-inch, M2, 2022)", + "Mac14,8": [ + "Mac Pro (2023)", + "Mac Pro (Rack, 2023)" + ], + "Mac14,9": "MacBook Pro (14-inch, 2023)", + "Mac14,10": "MacBook Pro (16-inch, 2023)", + "Mac14,12": "Mac mini (2023)", + "Mac14,13": "Mac Studio (2023)", + "Mac14,14": "Mac Studio (2023)", + "Mac14,15": "MacBook Air (15-inch, M2, 2023)", + "Mac15,3": "MacBook Pro (14-inch, Nov 2023)", + "Mac15,4": "iMac (24-inch, 2023, Two ports)", + "Mac15,5": "iMac (24-inch, 2023, Four ports)", + "Mac15,6": "MacBook Pro (14-inch, Nov 2023)", + "Mac15,7": "MacBook Pro (16-inch, Nov 2023)", + "Mac15,8": "MacBook Pro (14-inch, Nov 2023)", + "Mac15,9": "MacBook Pro (16-inch, Nov 2023)", + "Mac15,10": "MacBook Pro (14-inch, Nov 2023)", + "Mac15,11": "MacBook Pro (16-inch, Nov 2023)", + "Mac15,12": "MacBook Air (13-inch, M3, 2024)", + "Mac15,13": "MacBook Air (15-inch, M3, 2024)", + "Mac15,14": "Mac Studio (2025)", + "Mac16,1": "MacBook Pro (14-inch, 2024)", + "Mac16,2": "iMac (24-inch, 2024, Two ports)", + "Mac16,3": "iMac (24-inch, 2024, Four ports)", + "Mac16,5": "MacBook Pro (16-inch, 2024)", + "Mac16,6": "MacBook Pro (14-inch, 2024)", + "Mac16,7": "MacBook Pro (16-inch, 2024)", + "Mac16,8": "MacBook Pro (14-inch, 2024)", + "Mac16,9": "Mac Studio (2025)", + "Mac16,10": "Mac mini (2024)", + "Mac16,11": "Mac mini (2024)", + "Mac16,12": "MacBook Air (13-inch, M4, 2025)", + "Mac16,13": "MacBook Air (15-inch, M4, 2025)", + "Mac17,2": "MacBook Pro (14-inch, M5)", + "MacBook5,2": [ + "MacBook (13-inch, Early 2009)", + "MacBook (13-inch, Mid 2009)" + ], + "MacBook6,1": "MacBook (13-inch, Late 2009)", + "MacBook7,1": "MacBook (13-inch, Mid 2010)", + "MacBook8,1": "MacBook (Retina, 12-inch, Early 2015)", + "MacBook9,1": "MacBook (Retina, 12-inch, Early 2016)", + "MacBook10,1": "MacBook (Retina, 12-inch, 2017)", + "MacBookAir2,1": "MacBook Air (Mid 2009)", + "MacBookAir3,1": "MacBook Air (11-inch, Late 2010)", + "MacBookAir3,2": "MacBook Air (13-inch, Late 2010)", + "MacBookAir4,1": "MacBook Air (11-inch, Mid 2011)", + "MacBookAir4,2": "MacBook Air (13-inch, Mid 2011)", + "MacBookAir5,1": "MacBook Air (11-inch, Mid 2012)", + "MacBookAir5,2": "MacBook Air (13-inch, Mid 2012)", + "MacBookAir6,1": [ + "MacBook Air (11-inch, Early 2014)", + "MacBook Air (11-inch, Mid 2013)" + ], + "MacBookAir6,2": [ + "MacBook Air (13-inch, Early 2014)", + "MacBook Air (13-inch, Mid 2013)" + ], + "MacBookAir7,1": "MacBook Air (11-inch, Early 2015)", + "MacBookAir7,2": [ + "MacBook Air (13-inch, 2017)", + "MacBook Air (13-inch, Early 2015)" + ], + "MacBookAir8,1": "MacBook Air (Retina, 13-inch, 2018)", + "MacBookAir8,2": "MacBook Air (Retina, 13-inch, 2019)", + "MacBookAir9,1": "MacBook Air (Retina, 13-inch, 2020)", + "MacBookAir10,1": "MacBook Air (M1, 2020)", + "MacBookPro4,1": [ + "MacBook Pro (15-inch, Early 2008)", + "MacBook Pro (17-inch, Early 2008)" + ], + "MacBookPro5,1": "MacBook Pro (15-inch, Late 2008)", + "MacBookPro5,2": [ + "MacBook Pro (17-inch, Early 2009)", + "MacBook Pro (17-inch, Mid 2009)" + ], + "MacBookPro5,3": [ + "MacBook Pro (15-inch, 2.53GHz, Mid 2009)", + "MacBook Pro (15-inch, Mid 2009)" + ], + "MacBookPro5,5": "MacBook Pro (13-inch, Mid 2009)", + "MacBookPro6,1": "MacBook Pro (17-inch, Mid 2010)", + "MacBookPro6,2": "MacBook Pro (15-inch, Mid 2010)", + "MacBookPro7,1": "MacBook Pro (13-inch, Mid 2010)", + "MacBookPro8,1": [ + "MacBook Pro (13-inch, Early 2011)", + "MacBook Pro (13-inch, Late 2011)" + ], + "MacBookPro8,2": [ + "MacBook Pro (15-inch, Early 2011)", + "MacBook Pro (15-inch, Late 2011)" + ], + "MacBookPro8,3": [ + "MacBook Pro (17-inch, Early 2011)", + "MacBook Pro (17-inch, Late 2011)" + ], + "MacBookPro9,1": "MacBook Pro (15-inch, Mid 2012)", + "MacBookPro9,2": "MacBook Pro (13-inch, Mid 2012)", + "MacBookPro10,1": [ + "MacBook Pro (Retina, 15-inch, Early 2013)", + "MacBook Pro (Retina, 15-inch, Mid 2012)" + ], + "MacBookPro10,2": [ + "MacBook Pro (Retina, 13-inch, Early 2013)", + "MacBook Pro (Retina, 13-inch, Late 2012)" + ], + "MacBookPro11,1": [ + "MacBook Pro (Retina, 13-inch, Late 2013)", + "MacBook Pro (Retina, 13-inch, Mid 2014)" + ], + "MacBookPro11,2": [ + "MacBook Pro (Retina, 15-inch, Late 2013)", + "MacBook Pro (Retina, 15-inch, Mid 2014)" + ], + "MacBookPro11,3": [ + "MacBook Pro (Retina, 15-inch, Late 2013)", + "MacBook Pro (Retina, 15-inch, Mid 2014)" + ], + "MacBookPro11,4": "MacBook Pro (Retina, 15-inch, Mid 2015)", + "MacBookPro11,5": "MacBook Pro (Retina, 15-inch, Mid 2015)", + "MacBookPro12,1": "MacBook Pro (Retina, 13-inch, Early 2015)", + "MacBookPro13,1": "MacBook Pro (13-inch, 2016, Two Thunderbolt 3 ports)", + "MacBookPro13,2": "MacBook Pro (13-inch, 2016, Four Thunderbolt 3 ports)", + "MacBookPro13,3": "MacBook Pro (15-inch, 2016)", + "MacBookPro14,1": "MacBook Pro (13-inch, 2017, Two Thunderbolt 3 ports)", + "MacBookPro14,2": "MacBook Pro (13-inch, 2017, Four Thunderbolt 3 ports)", + "MacBookPro14,3": "MacBook Pro (15-inch, 2017)", + "MacBookPro15,1": [ + "MacBook Pro (15-inch, 2018)", + "MacBook Pro (15-inch, 2019)" + ], + "MacBookPro15,2": [ + "MacBook Pro (13-inch, 2018, Four Thunderbolt 3 ports)", + "MacBook Pro (13-inch, 2019, Four Thunderbolt 3 ports)" + ], + "MacBookPro15,3": "MacBook Pro (15-inch, 2019)", + "MacBookPro15,4": "MacBook Pro (13-inch, 2019, Two Thunderbolt 3 ports)", + "MacBookPro16,1": "MacBook Pro (16-inch, 2019)", + "MacBookPro16,2": "MacBook Pro (13-inch, 2020, Four Thunderbolt 3 ports)", + "MacBookPro16,3": "MacBook Pro (13-inch, 2020, Two Thunderbolt 3 ports)", + "MacBookPro16,4": "MacBook Pro (16-inch, 2019)", + "MacBookPro17,1": "MacBook Pro (13-inch, M1, 2020)", + "MacBookPro18,1": "MacBook Pro (16-inch, 2021)", + "MacBookPro18,2": "MacBook Pro (16-inch, 2021)", + "MacBookPro18,3": "MacBook Pro (14-inch, 2021)", + "MacBookPro18,4": "MacBook Pro (14-inch, 2021)", + "Macmini3,1": [ + "Mac mini (Early 2009)", + "Mac mini (Late 2009)" + ], + "Macmini4,1": "Mac mini (Mid 2010)", + "Macmini5,1": "Mac mini (Mid 2011)", + "Macmini5,2": "Mac mini (Mid 2011)", + "Macmini6,1": "Mac mini (Late 2012)", + "Macmini6,2": "Mac mini (Late 2012)", + "Macmini7,1": "Mac mini (Late 2014)", + "Macmini8,1": "Mac mini (2018)", + "Macmini9,1": "Mac mini (M1, 2020)", + "MacPro4,1": "Mac Pro (Early 2009)", + "MacPro5,1": [ + "Mac Pro (Mid 2010)", + "Mac Pro (Mid 2012)", + "Mac Pro Server (Mid 2010)", + "Mac Pro Server (Mid 2012)" + ], + "MacPro6,1": "Mac Pro (Late 2013)", + "MacPro7,1": [ + "Mac Pro (2019)", + "Mac Pro (Rack, 2019)" + ] +} diff --git a/apps/macos/Sources/OpenClaw/Resources/Info.plist b/apps/macos/Sources/OpenClaw/Resources/Info.plist new file mode 100644 index 0000000000000000000000000000000000000000..43854bcb12cb157e079cfb19eebf6ef181ecc361 --- /dev/null +++ b/apps/macos/Sources/OpenClaw/Resources/Info.plist @@ -0,0 +1,79 @@ + + + + + CFBundleDevelopmentRegion + en + CFBundleExecutable + OpenClaw + CFBundleIdentifier + ai.openclaw.mac + CFBundleInfoDictionaryVersion + 6.0 + CFBundleName + OpenClaw + CFBundlePackageType + APPL + CFBundleShortVersionString + 2026.1.30 + CFBundleVersion + 202601290 + CFBundleIconFile + OpenClaw + CFBundleURLTypes + + + CFBundleURLName + ai.openclaw.mac.deeplink + CFBundleURLSchemes + + openclaw + + + + LSMinimumSystemVersion + 15.0 + LSUIElement + + + OpenClawBuildTimestamp + + OpenClawGitCommit + + + NSUserNotificationUsageDescription + OpenClaw needs notification permission to show alerts for agent actions. + NSScreenCaptureDescription + OpenClaw captures the screen when the agent needs screenshots for context. + NSCameraUsageDescription + OpenClaw can capture photos or short video clips when requested by the agent. + NSLocationUsageDescription + OpenClaw can share your location when requested by the agent. + NSLocationWhenInUseUsageDescription + OpenClaw can share your location when requested by the agent. + NSLocationAlwaysAndWhenInUseUsageDescription + OpenClaw can share your location when requested by the agent. + NSMicrophoneUsageDescription + OpenClaw needs the mic for Voice Wake tests and agent audio capture. + NSSpeechRecognitionUsageDescription + OpenClaw uses speech recognition to detect your Voice Wake trigger phrase. + NSAppleEventsUsageDescription + OpenClaw needs Automation (AppleScript) permission to drive Terminal and other apps for agent actions. + + NSAppTransportSecurity + + NSAllowsArbitraryLoadsInWebContent + + NSExceptionDomains + + 100.100.100.100 + + NSExceptionAllowsInsecureHTTPLoads + + NSIncludesSubdomains + + + + + + diff --git a/apps/macos/Sources/OpenClaw/Resources/OpenClaw.icns b/apps/macos/Sources/OpenClaw/Resources/OpenClaw.icns new file mode 100644 index 0000000000000000000000000000000000000000..de29f9920cf79044e7e61def3db2a226a7f57a14 --- /dev/null +++ b/apps/macos/Sources/OpenClaw/Resources/OpenClaw.icns @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1caf8c48db04e1abb3d327f91f4a363762d31bdcc993b498696ffd5233b52bec +size 1884490 diff --git a/apps/macos/Sources/OpenClaw/RuntimeLocator.swift b/apps/macos/Sources/OpenClaw/RuntimeLocator.swift new file mode 100644 index 0000000000000000000000000000000000000000..8ec23a067be98e56e26c9b52f55d38093412d333 --- /dev/null +++ b/apps/macos/Sources/OpenClaw/RuntimeLocator.swift @@ -0,0 +1,167 @@ +import Foundation +import OSLog + +enum RuntimeKind: String { + case node +} + +struct RuntimeVersion: Comparable, CustomStringConvertible { + let major: Int + let minor: Int + let patch: Int + + var description: String { "\(self.major).\(self.minor).\(self.patch)" } + + static func < (lhs: RuntimeVersion, rhs: RuntimeVersion) -> Bool { + if lhs.major != rhs.major { return lhs.major < rhs.major } + if lhs.minor != rhs.minor { return lhs.minor < rhs.minor } + return lhs.patch < rhs.patch + } + + static func from(string: String) -> RuntimeVersion? { + // Accept optional leading "v" and ignore trailing metadata. + let pattern = #"(\d+)\.(\d+)\.(\d+)"# + guard let match = string.range(of: pattern, options: .regularExpression) else { return nil } + let versionString = String(string[match]) + let parts = versionString.split(separator: ".") + guard parts.count == 3, + let major = Int(parts[0]), + let minor = Int(parts[1]), + let patch = Int(parts[2]) + else { return nil } + return RuntimeVersion(major: major, minor: minor, patch: patch) + } +} + +struct RuntimeResolution { + let kind: RuntimeKind + let path: String + let version: RuntimeVersion +} + +enum RuntimeResolutionError: Error { + case notFound(searchPaths: [String]) + case unsupported( + kind: RuntimeKind, + found: RuntimeVersion, + required: RuntimeVersion, + path: String, + searchPaths: [String]) + case versionParse(kind: RuntimeKind, raw: String, path: String, searchPaths: [String]) +} + +enum RuntimeLocator { + private static let logger = Logger(subsystem: "ai.openclaw", category: "runtime") + private static let minNode = RuntimeVersion(major: 22, minor: 0, patch: 0) + + static func resolve( + searchPaths: [String] = CommandResolver.preferredPaths()) -> Result + { + let pathEnv = searchPaths.joined(separator: ":") + let runtime: RuntimeKind = .node + + guard let binary = findExecutable(named: runtime.binaryName, searchPaths: searchPaths) else { + return .failure(.notFound(searchPaths: searchPaths)) + } + guard let rawVersion = readVersion(of: binary, pathEnv: pathEnv) else { + return .failure(.versionParse( + kind: runtime, + raw: "(unreadable)", + path: binary, + searchPaths: searchPaths)) + } + guard let parsed = RuntimeVersion.from(string: rawVersion) else { + return .failure(.versionParse(kind: runtime, raw: rawVersion, path: binary, searchPaths: searchPaths)) + } + guard parsed >= self.minNode else { + return .failure(.unsupported( + kind: runtime, + found: parsed, + required: self.minNode, + path: binary, + searchPaths: searchPaths)) + } + + return .success(RuntimeResolution(kind: runtime, path: binary, version: parsed)) + } + + static func describeFailure(_ error: RuntimeResolutionError) -> String { + switch error { + case let .notFound(searchPaths): + [ + "openclaw needs Node >=22.0.0 but found no runtime.", + "PATH searched: \(searchPaths.joined(separator: ":"))", + "Install Node: https://nodejs.org/en/download", + ].joined(separator: "\n") + case let .unsupported(kind, found, required, path, searchPaths): + [ + "Found \(kind.rawValue) \(found) at \(path) but need >= \(required).", + "PATH searched: \(searchPaths.joined(separator: ":"))", + "Upgrade Node and rerun openclaw.", + ].joined(separator: "\n") + case let .versionParse(kind, raw, path, searchPaths): + [ + "Could not parse \(kind.rawValue) version output \"\(raw)\" from \(path).", + "PATH searched: \(searchPaths.joined(separator: ":"))", + "Try reinstalling or pinning a supported version (Node >=22.0.0).", + ].joined(separator: "\n") + } + } + + // MARK: - Internals + + private static func findExecutable(named name: String, searchPaths: [String]) -> String? { + let fm = FileManager() + for dir in searchPaths { + let candidate = (dir as NSString).appendingPathComponent(name) + if fm.isExecutableFile(atPath: candidate) { + return candidate + } + } + return nil + } + + private static func readVersion(of binary: String, pathEnv: String) -> String? { + let start = Date() + let process = Process() + process.executableURL = URL(fileURLWithPath: binary) + process.arguments = ["--version"] + process.environment = ["PATH": pathEnv] + + let pipe = Pipe() + process.standardOutput = pipe + process.standardError = pipe + + do { + let data = try process.runAndReadToEnd(from: pipe) + let elapsedMs = Int(Date().timeIntervalSince(start) * 1000) + if elapsedMs > 500 { + self.logger.warning( + """ + runtime --version slow (\(elapsedMs, privacy: .public)ms) \ + bin=\(binary, privacy: .public) + """) + } else { + self.logger.debug( + """ + runtime --version ok (\(elapsedMs, privacy: .public)ms) \ + bin=\(binary, privacy: .public) + """) + } + return String(data: data, encoding: .utf8)?.trimmingCharacters(in: .whitespacesAndNewlines) + } catch { + let elapsedMs = Int(Date().timeIntervalSince(start) * 1000) + self.logger.error( + """ + runtime --version failed (\(elapsedMs, privacy: .public)ms) \ + bin=\(binary, privacy: .public) \ + err=\(error.localizedDescription, privacy: .public) + """) + return nil + } + } +} + +extension RuntimeKind { + fileprivate var binaryName: String { "node" } +} diff --git a/apps/macos/Sources/OpenClaw/ScreenRecordService.swift b/apps/macos/Sources/OpenClaw/ScreenRecordService.swift new file mode 100644 index 0000000000000000000000000000000000000000..30d854b114784d9e483110914eaa549488579b5d --- /dev/null +++ b/apps/macos/Sources/OpenClaw/ScreenRecordService.swift @@ -0,0 +1,266 @@ +import AVFoundation +import Foundation +import OSLog +@preconcurrency import ScreenCaptureKit + +@MainActor +final class ScreenRecordService { + enum ScreenRecordError: LocalizedError { + case noDisplays + case invalidScreenIndex(Int) + case noFramesCaptured + case writeFailed(String) + + var errorDescription: String? { + switch self { + case .noDisplays: + "No displays available for screen recording" + case let .invalidScreenIndex(idx): + "Invalid screen index \(idx)" + case .noFramesCaptured: + "No frames captured" + case let .writeFailed(msg): + msg + } + } + } + + private let logger = Logger(subsystem: "ai.openclaw", category: "screenRecord") + + func record( + screenIndex: Int?, + durationMs: Int?, + fps: Double?, + includeAudio: Bool?, + outPath: String?) async throws -> (path: String, hasAudio: Bool) + { + let durationMs = Self.clampDurationMs(durationMs) + let fps = Self.clampFps(fps) + let includeAudio = includeAudio ?? false + + let outURL: URL = { + if let outPath, !outPath.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty { + return URL(fileURLWithPath: outPath) + } + return FileManager().temporaryDirectory + .appendingPathComponent("openclaw-screen-record-\(UUID().uuidString).mp4") + }() + try? FileManager().removeItem(at: outURL) + + let content = try await SCShareableContent.current + let displays = content.displays.sorted { $0.displayID < $1.displayID } + guard !displays.isEmpty else { throw ScreenRecordError.noDisplays } + + let idx = screenIndex ?? 0 + guard idx >= 0, idx < displays.count else { throw ScreenRecordError.invalidScreenIndex(idx) } + let display = displays[idx] + + let filter = SCContentFilter(display: display, excludingWindows: []) + let config = SCStreamConfiguration() + config.width = display.width + config.height = display.height + config.queueDepth = 8 + config.showsCursor = true + config.minimumFrameInterval = CMTime(value: 1, timescale: CMTimeScale(max(1, Int32(fps.rounded())))) + if includeAudio { + config.capturesAudio = true + } + + let recorder = try StreamRecorder( + outputURL: outURL, + width: display.width, + height: display.height, + includeAudio: includeAudio, + logger: self.logger) + + let stream = SCStream(filter: filter, configuration: config, delegate: recorder) + try stream.addStreamOutput(recorder, type: .screen, sampleHandlerQueue: recorder.queue) + if includeAudio { + try stream.addStreamOutput(recorder, type: .audio, sampleHandlerQueue: recorder.queue) + } + + self.logger.info( + "screen record start idx=\(idx) durationMs=\(durationMs) fps=\(fps) out=\(outURL.path, privacy: .public)") + + var started = false + do { + try await stream.startCapture() + started = true + try await Task.sleep(nanoseconds: UInt64(durationMs) * 1_000_000) + try await stream.stopCapture() + } catch { + if started { try? await stream.stopCapture() } + throw error + } + + try await recorder.finish() + return (path: outURL.path, hasAudio: recorder.hasAudio) + } + + private nonisolated static func clampDurationMs(_ ms: Int?) -> Int { + let v = ms ?? 10000 + return min(60000, max(250, v)) + } + + private nonisolated static func clampFps(_ fps: Double?) -> Double { + let v = fps ?? 10 + if !v.isFinite { return 10 } + return min(60, max(1, v)) + } +} + +private final class StreamRecorder: NSObject, SCStreamOutput, SCStreamDelegate, @unchecked Sendable { + let queue = DispatchQueue(label: "ai.openclaw.screenRecord.writer") + + private let logger: Logger + private let writer: AVAssetWriter + private let input: AVAssetWriterInput + private let audioInput: AVAssetWriterInput? + let hasAudio: Bool + + private var started = false + private var sawFrame = false + private var didFinish = false + private var pendingErrorMessage: String? + + init(outputURL: URL, width: Int, height: Int, includeAudio: Bool, logger: Logger) throws { + self.logger = logger + self.writer = try AVAssetWriter(outputURL: outputURL, fileType: .mp4) + + let settings: [String: Any] = [ + AVVideoCodecKey: AVVideoCodecType.h264, + AVVideoWidthKey: width, + AVVideoHeightKey: height, + ] + self.input = AVAssetWriterInput(mediaType: .video, outputSettings: settings) + self.input.expectsMediaDataInRealTime = true + + guard self.writer.canAdd(self.input) else { + throw ScreenRecordService.ScreenRecordError.writeFailed("Cannot add video input") + } + self.writer.add(self.input) + + if includeAudio { + let audioSettings: [String: Any] = [ + AVFormatIDKey: kAudioFormatMPEG4AAC, + AVNumberOfChannelsKey: 1, + AVSampleRateKey: 44100, + AVEncoderBitRateKey: 96000, + ] + let audioInput = AVAssetWriterInput(mediaType: .audio, outputSettings: audioSettings) + audioInput.expectsMediaDataInRealTime = true + if self.writer.canAdd(audioInput) { + self.writer.add(audioInput) + self.audioInput = audioInput + self.hasAudio = true + } else { + self.audioInput = nil + self.hasAudio = false + } + } else { + self.audioInput = nil + self.hasAudio = false + } + super.init() + } + + func stream(_ stream: SCStream, didStopWithError error: any Error) { + self.queue.async { + let msg = String(describing: error) + self.pendingErrorMessage = msg + self.logger.error("screen record stream stopped with error: \(msg, privacy: .public)") + _ = stream + } + } + + func stream( + _ stream: SCStream, + didOutputSampleBuffer sampleBuffer: CMSampleBuffer, + of type: SCStreamOutputType) + { + guard CMSampleBufferDataIsReady(sampleBuffer) else { return } + // Callback runs on `sampleHandlerQueue` (`self.queue`). + switch type { + case .screen: + self.handleVideo(sampleBuffer: sampleBuffer) + case .audio: + self.handleAudio(sampleBuffer: sampleBuffer) + case .microphone: + break + @unknown default: + break + } + _ = stream + } + + private func handleVideo(sampleBuffer: CMSampleBuffer) { + if let msg = self.pendingErrorMessage { + self.logger.error("screen record aborting due to prior error: \(msg, privacy: .public)") + return + } + if self.didFinish { return } + + if !self.started { + guard self.writer.startWriting() else { + self.pendingErrorMessage = self.writer.error?.localizedDescription ?? "Failed to start writer" + return + } + let pts = CMSampleBufferGetPresentationTimeStamp(sampleBuffer) + self.writer.startSession(atSourceTime: pts) + self.started = true + } + + self.sawFrame = true + if self.input.isReadyForMoreMediaData { + _ = self.input.append(sampleBuffer) + } + } + + private func handleAudio(sampleBuffer: CMSampleBuffer) { + guard let audioInput else { return } + if let msg = self.pendingErrorMessage { + self.logger.error("screen record audio aborting due to prior error: \(msg, privacy: .public)") + return + } + if self.didFinish || !self.started { return } + if audioInput.isReadyForMoreMediaData { + _ = audioInput.append(sampleBuffer) + } + } + + func finish() async throws { + try await withCheckedThrowingContinuation { (cont: CheckedContinuation) in + self.queue.async { + if let msg = self.pendingErrorMessage { + cont.resume(throwing: ScreenRecordService.ScreenRecordError.writeFailed(msg)) + return + } + guard self.started, self.sawFrame else { + cont.resume(throwing: ScreenRecordService.ScreenRecordError.noFramesCaptured) + return + } + if self.didFinish { + cont.resume() + return + } + self.didFinish = true + + self.input.markAsFinished() + self.audioInput?.markAsFinished() + self.writer.finishWriting { + if let err = self.writer.error { + cont + .resume(throwing: ScreenRecordService.ScreenRecordError + .writeFailed(err.localizedDescription)) + } else if self.writer.status != .completed { + cont + .resume(throwing: ScreenRecordService.ScreenRecordError + .writeFailed("Failed to finalize video")) + } else { + cont.resume() + } + } + } + } + } +} diff --git a/apps/macos/Sources/OpenClaw/ScreenshotSize.swift b/apps/macos/Sources/OpenClaw/ScreenshotSize.swift new file mode 100644 index 0000000000000000000000000000000000000000..e1ad915f58ac2c36a68bd0214d41faa6c86ed570 --- /dev/null +++ b/apps/macos/Sources/OpenClaw/ScreenshotSize.swift @@ -0,0 +1,17 @@ +import Foundation +import ImageIO + +enum ScreenshotSize { + struct Size { + let width: Int + let height: Int + } + + static func readPNGSize(data: Data) -> Size? { + guard let source = CGImageSourceCreateWithData(data as CFData, nil) else { return nil } + guard let props = CGImageSourceCopyPropertiesAtIndex(source, 0, nil) as? [CFString: Any] else { return nil } + guard let width = props[kCGImagePropertyPixelWidth] as? Int else { return nil } + guard let height = props[kCGImagePropertyPixelHeight] as? Int else { return nil } + return Size(width: width, height: height) + } +} diff --git a/apps/macos/Sources/OpenClaw/SessionActions.swift b/apps/macos/Sources/OpenClaw/SessionActions.swift new file mode 100644 index 0000000000000000000000000000000000000000..10a3c7641d4f5ed939f98eb4ae85d0313326d336 --- /dev/null +++ b/apps/macos/Sources/OpenClaw/SessionActions.swift @@ -0,0 +1,91 @@ +import AppKit +import Foundation + +enum SessionActions { + static func patchSession( + key: String, + thinking: String?? = nil, + verbose: String?? = nil) async throws + { + var params: [String: AnyHashable] = ["key": AnyHashable(key)] + + if let thinking { + params["thinkingLevel"] = thinking.map(AnyHashable.init) ?? AnyHashable(NSNull()) + } + if let verbose { + params["verboseLevel"] = verbose.map(AnyHashable.init) ?? AnyHashable(NSNull()) + } + + _ = try await ControlChannel.shared.request(method: "sessions.patch", params: params) + } + + static func resetSession(key: String) async throws { + _ = try await ControlChannel.shared.request( + method: "sessions.reset", + params: ["key": AnyHashable(key)]) + } + + static func deleteSession(key: String) async throws { + _ = try await ControlChannel.shared.request( + method: "sessions.delete", + params: ["key": AnyHashable(key), "deleteTranscript": AnyHashable(true)]) + } + + static func compactSession(key: String, maxLines: Int = 400) async throws { + _ = try await ControlChannel.shared.request( + method: "sessions.compact", + params: ["key": AnyHashable(key), "maxLines": AnyHashable(maxLines)]) + } + + @MainActor + static func confirmDestructiveAction(title: String, message: String, action: String) -> Bool { + let alert = NSAlert() + alert.messageText = title + alert.informativeText = message + alert.addButton(withTitle: action) + alert.addButton(withTitle: "Cancel") + alert.alertStyle = .warning + return alert.runModal() == .alertFirstButtonReturn + } + + @MainActor + static func presentError(title: String, error: Error) { + let alert = NSAlert() + alert.messageText = title + alert.informativeText = (error as? LocalizedError)?.errorDescription ?? error.localizedDescription + alert.addButton(withTitle: "OK") + alert.alertStyle = .warning + alert.runModal() + } + + @MainActor + static func openSessionLogInCode(sessionId: String, storePath: String?) { + let candidates: [URL] = { + var urls: [URL] = [] + if let storePath, !storePath.isEmpty { + let dir = URL(fileURLWithPath: storePath).deletingLastPathComponent() + urls.append(dir.appendingPathComponent("\(sessionId).jsonl")) + } + urls.append(OpenClawPaths.stateDirURL.appendingPathComponent("sessions/\(sessionId).jsonl")) + return urls + }() + + let existing = candidates.first(where: { FileManager().fileExists(atPath: $0.path) }) + guard let url = existing else { + let alert = NSAlert() + alert.messageText = "Session log not found" + alert.informativeText = sessionId + alert.runModal() + return + } + + let proc = Process() + proc.launchPath = "/usr/bin/env" + proc.arguments = ["code", url.path] + if (try? proc.run()) != nil { + return + } + + NSWorkspace.shared.activateFileViewerSelecting([url]) + } +} diff --git a/apps/macos/Sources/OpenClaw/SessionData.swift b/apps/macos/Sources/OpenClaw/SessionData.swift new file mode 100644 index 0000000000000000000000000000000000000000..a106cf9dc655f517cde11fa4e0b0f2c24374d1dd --- /dev/null +++ b/apps/macos/Sources/OpenClaw/SessionData.swift @@ -0,0 +1,341 @@ +import Foundation +import SwiftUI + +struct GatewaySessionDefaultsRecord: Codable { + let model: String? + let contextTokens: Int? +} + +struct GatewaySessionEntryRecord: Codable { + let key: String + let displayName: String? + let provider: String? + let subject: String? + let room: String? + let space: String? + let updatedAt: Double? + let sessionId: String? + let systemSent: Bool? + let abortedLastRun: Bool? + let thinkingLevel: String? + let verboseLevel: String? + let inputTokens: Int? + let outputTokens: Int? + let totalTokens: Int? + let model: String? + let contextTokens: Int? +} + +struct GatewaySessionsListResponse: Codable { + let ts: Double? + let path: String + let count: Int + let defaults: GatewaySessionDefaultsRecord? + let sessions: [GatewaySessionEntryRecord] +} + +struct SessionTokenStats { + let input: Int + let output: Int + let total: Int + let contextTokens: Int + + var contextSummaryShort: String { + "\(Self.formatKTokens(self.total))/\(Self.formatKTokens(self.contextTokens))" + } + + var percentUsed: Int? { + guard self.contextTokens > 0, self.total > 0 else { return nil } + return min(100, Int(round((Double(self.total) / Double(self.contextTokens)) * 100))) + } + + var summary: String { + let parts = ["in \(input)", "out \(output)", "total \(total)"] + var text = parts.joined(separator: " | ") + if let percentUsed { + text += " (\(percentUsed)% of \(self.contextTokens))" + } + return text + } + + static func formatKTokens(_ value: Int) -> String { + if value < 1000 { return "\(value)" } + let thousands = Double(value) / 1000 + let decimals = value >= 10000 ? 0 : 1 + return String(format: "%.\(decimals)fk", thousands) + } +} + +struct SessionRow: Identifiable { + let id: String + let key: String + let kind: SessionKind + let displayName: String? + let provider: String? + let subject: String? + let room: String? + let space: String? + let updatedAt: Date? + let sessionId: String? + let thinkingLevel: String? + let verboseLevel: String? + let systemSent: Bool + let abortedLastRun: Bool + let tokens: SessionTokenStats + let model: String? + + var ageText: String { relativeAge(from: self.updatedAt) } + var label: String { self.displayName ?? self.key } + + var flagLabels: [String] { + var flags: [String] = [] + if let thinkingLevel { flags.append("think \(thinkingLevel)") } + if let verboseLevel { flags.append("verbose \(verboseLevel)") } + if self.systemSent { flags.append("system sent") } + if self.abortedLastRun { flags.append("aborted") } + return flags + } +} + +enum SessionKind { + case direct, group, global, unknown + + static func from(key: String) -> SessionKind { + if key == "global" { return .global } + if key.hasPrefix("group:") { return .group } + if key.contains(":group:") { return .group } + if key.contains(":channel:") { return .group } + if key == "unknown" { return .unknown } + return .direct + } + + var label: String { + switch self { + case .direct: "Direct" + case .group: "Group" + case .global: "Global" + case .unknown: "Unknown" + } + } + + var tint: Color { + switch self { + case .direct: .accentColor + case .group: .orange + case .global: .purple + case .unknown: .gray + } + } +} + +struct SessionDefaults { + let model: String + let contextTokens: Int +} + +extension SessionRow { + static var previewRows: [SessionRow] { + [ + SessionRow( + id: "direct-1", + key: "user@example.com", + kind: .direct, + displayName: nil, + provider: nil, + subject: nil, + room: nil, + space: nil, + updatedAt: Date().addingTimeInterval(-90), + sessionId: "sess-direct-1234", + thinkingLevel: "low", + verboseLevel: "info", + systemSent: false, + abortedLastRun: false, + tokens: SessionTokenStats(input: 320, output: 680, total: 1000, contextTokens: 200_000), + model: "claude-3.5-sonnet"), + SessionRow( + id: "group-1", + key: "discord:channel:release-squad", + kind: .group, + displayName: "discord:#release-squad", + provider: "discord", + subject: nil, + room: "#release-squad", + space: nil, + updatedAt: Date().addingTimeInterval(-3600), + sessionId: "sess-group-4321", + thinkingLevel: "medium", + verboseLevel: nil, + systemSent: true, + abortedLastRun: true, + tokens: SessionTokenStats(input: 5000, output: 1200, total: 6200, contextTokens: 200_000), + model: "claude-opus-4-5"), + SessionRow( + id: "global", + key: "global", + kind: .global, + displayName: nil, + provider: nil, + subject: nil, + room: nil, + space: nil, + updatedAt: Date().addingTimeInterval(-86400), + sessionId: nil, + thinkingLevel: nil, + verboseLevel: nil, + systemSent: false, + abortedLastRun: false, + tokens: SessionTokenStats(input: 150, output: 220, total: 370, contextTokens: 200_000), + model: "gpt-4.1-mini"), + ] + } +} + +struct ModelChoice: Identifiable, Hashable, Codable { + let id: String + let name: String + let provider: String + let contextWindow: Int? +} + +extension String? { + var isNilOrEmpty: Bool { + switch self { + case .none: true + case let .some(value): value.isEmpty + } + } +} + +extension [String] { + fileprivate func dedupedPreserveOrder() -> [String] { + var seen = Set() + var result: [String] = [] + for item in self where !seen.contains(item) { + seen.insert(item) + result.append(item) + } + return result + } +} + +enum SessionLoadError: LocalizedError { + case gatewayUnavailable(String) + case decodeFailed(String) + + var errorDescription: String? { + switch self { + case let .gatewayUnavailable(reason): + "Could not reach the gateway for sessions: \(reason)" + + case let .decodeFailed(reason): + "Could not decode gateway session payload: \(reason)" + } + } +} + +struct SessionStoreSnapshot { + let storePath: String + let defaults: SessionDefaults + let rows: [SessionRow] +} + +@MainActor +enum SessionLoader { + static let fallbackModel = "claude-opus-4-5" + static let fallbackContextTokens = 200_000 + + static let defaultStorePath = standardize( + OpenClawPaths.stateDirURL + .appendingPathComponent("sessions/sessions.json").path) + + static func loadSnapshot( + activeMinutes: Int? = nil, + limit: Int? = nil, + includeGlobal: Bool = true, + includeUnknown: Bool = true) async throws -> SessionStoreSnapshot + { + var params: [String: AnyHashable] = [ + "includeGlobal": AnyHashable(includeGlobal), + "includeUnknown": AnyHashable(includeUnknown), + ] + if let activeMinutes { params["activeMinutes"] = AnyHashable(activeMinutes) } + if let limit { params["limit"] = AnyHashable(limit) } + + let data: Data + do { + data = try await ControlChannel.shared.request(method: "sessions.list", params: params) + } catch { + let msg = (error as? LocalizedError)?.errorDescription ?? error.localizedDescription + if msg.localizedCaseInsensitiveContains("unknown method: sessions.list") { + throw SessionLoadError.gatewayUnavailable( + "Gateway is too old (missing sessions.list). Restart/update the gateway.") + } + throw SessionLoadError.gatewayUnavailable(msg) + } + + let decoded: GatewaySessionsListResponse + do { + decoded = try JSONDecoder().decode(GatewaySessionsListResponse.self, from: data) + } catch { + throw SessionLoadError.decodeFailed(error.localizedDescription) + } + + let defaults = SessionDefaults( + model: decoded.defaults?.model ?? self.fallbackModel, + contextTokens: decoded.defaults?.contextTokens ?? self.fallbackContextTokens) + + let rows = decoded.sessions.map { entry -> SessionRow in + let updated = entry.updatedAt.map { Date(timeIntervalSince1970: $0 / 1000) } + let input = entry.inputTokens ?? 0 + let output = entry.outputTokens ?? 0 + let total = entry.totalTokens ?? input + output + let context = entry.contextTokens ?? defaults.contextTokens + let model = entry.model ?? defaults.model + + return SessionRow( + id: entry.key, + key: entry.key, + kind: SessionKind.from(key: entry.key), + displayName: entry.displayName, + provider: entry.provider, + subject: entry.subject, + room: entry.room, + space: entry.space, + updatedAt: updated, + sessionId: entry.sessionId, + thinkingLevel: entry.thinkingLevel, + verboseLevel: entry.verboseLevel, + systemSent: entry.systemSent ?? false, + abortedLastRun: entry.abortedLastRun ?? false, + tokens: SessionTokenStats( + input: input, + output: output, + total: total, + contextTokens: context), + model: model) + }.sorted { ($0.updatedAt ?? .distantPast) > ($1.updatedAt ?? .distantPast) } + + return SessionStoreSnapshot(storePath: decoded.path, defaults: defaults, rows: rows) + } + + static func loadRows() async throws -> [SessionRow] { + try await self.loadSnapshot().rows + } + + private static func standardize(_ path: String) -> String { + (path as NSString).expandingTildeInPath.replacingOccurrences(of: "//", with: "/") + } +} + +func relativeAge(from date: Date?) -> String { + guard let date else { return "unknown" } + let delta = Date().timeIntervalSince(date) + if delta < 60 { return "just now" } + let minutes = Int(round(delta / 60)) + if minutes < 60 { return "\(minutes)m ago" } + let hours = Int(round(Double(minutes) / 60)) + if hours < 48 { return "\(hours)h ago" } + let days = Int(round(Double(hours) / 24)) + return "\(days)d ago" +} diff --git a/apps/macos/Sources/OpenClaw/SessionMenuLabelView.swift b/apps/macos/Sources/OpenClaw/SessionMenuLabelView.swift new file mode 100644 index 0000000000000000000000000000000000000000..1cbeedd392d6db6837c0711a8fbce595d03da4a6 --- /dev/null +++ b/apps/macos/Sources/OpenClaw/SessionMenuLabelView.swift @@ -0,0 +1,65 @@ +import SwiftUI + +private struct MenuItemHighlightedKey: EnvironmentKey { + static let defaultValue = false +} + +extension EnvironmentValues { + var menuItemHighlighted: Bool { + get { self[MenuItemHighlightedKey.self] } + set { self[MenuItemHighlightedKey.self] = newValue } + } +} + +struct SessionMenuLabelView: View { + let row: SessionRow + let width: CGFloat + @Environment(\.menuItemHighlighted) private var isHighlighted + private let paddingLeading: CGFloat = 22 + private let paddingTrailing: CGFloat = 14 + private let barHeight: CGFloat = 6 + + private var primaryTextColor: Color { + self.isHighlighted ? Color(nsColor: .selectedMenuItemTextColor) : .primary + } + + private var secondaryTextColor: Color { + self.isHighlighted ? Color(nsColor: .selectedMenuItemTextColor).opacity(0.85) : .secondary + } + + var body: some View { + VStack(alignment: .leading, spacing: 8) { + ContextUsageBar( + usedTokens: self.row.tokens.total, + contextTokens: self.row.tokens.contextTokens, + width: max(1, self.width - (self.paddingLeading + self.paddingTrailing)), + height: self.barHeight) + + HStack(alignment: .firstTextBaseline, spacing: 2) { + Text(self.row.label) + .font(.caption.weight(self.row.key == "main" ? .semibold : .regular)) + .foregroundStyle(self.primaryTextColor) + .lineLimit(1) + .truncationMode(.middle) + .layoutPriority(1) + + Spacer(minLength: 4) + + Text("\(self.row.tokens.contextSummaryShort) · \(self.row.ageText)") + .font(.caption.monospacedDigit()) + .foregroundStyle(self.secondaryTextColor) + .lineLimit(1) + .fixedSize(horizontal: true, vertical: false) + .layoutPriority(2) + + Image(systemName: "chevron.right") + .font(.caption.weight(.semibold)) + .foregroundStyle(self.secondaryTextColor) + .padding(.leading, 2) + } + } + .padding(.vertical, 10) + .padding(.leading, self.paddingLeading) + .padding(.trailing, self.paddingTrailing) + } +} diff --git a/apps/macos/Sources/OpenClaw/SessionMenuPreviewView.swift b/apps/macos/Sources/OpenClaw/SessionMenuPreviewView.swift new file mode 100644 index 0000000000000000000000000000000000000000..dc129df9f41e834081c37871c12a7f6632eb45c4 --- /dev/null +++ b/apps/macos/Sources/OpenClaw/SessionMenuPreviewView.swift @@ -0,0 +1,495 @@ +import OpenClawChatUI +import OpenClawKit +import OpenClawProtocol +import OSLog +import SwiftUI + +struct SessionPreviewItem: Identifiable, Sendable { + let id: String + let role: PreviewRole + let text: String +} + +enum PreviewRole: String, Sendable { + case user + case assistant + case tool + case system + case other + + var label: String { + switch self { + case .user: "User" + case .assistant: "Agent" + case .tool: "Tool" + case .system: "System" + case .other: "Other" + } + } +} + +actor SessionPreviewCache { + static let shared = SessionPreviewCache() + + private struct CacheEntry { + let snapshot: SessionMenuPreviewSnapshot + let updatedAt: Date + } + + private var entries: [String: CacheEntry] = [:] + + func cachedSnapshot(for sessionKey: String, maxAge: TimeInterval) -> SessionMenuPreviewSnapshot? { + guard let entry = self.entries[sessionKey] else { return nil } + guard Date().timeIntervalSince(entry.updatedAt) < maxAge else { return nil } + return entry.snapshot + } + + func store(snapshot: SessionMenuPreviewSnapshot, for sessionKey: String) { + self.entries[sessionKey] = CacheEntry(snapshot: snapshot, updatedAt: Date()) + } + + func lastSnapshot(for sessionKey: String) -> SessionMenuPreviewSnapshot? { + self.entries[sessionKey]?.snapshot + } +} + +actor SessionPreviewLimiter { + static let shared = SessionPreviewLimiter(maxConcurrent: 2) + + private let maxConcurrent: Int + private var available: Int + private var waitQueue: [UUID] = [] + private var waiters: [UUID: CheckedContinuation] = [:] + + init(maxConcurrent: Int) { + let normalized = max(1, maxConcurrent) + self.maxConcurrent = normalized + self.available = normalized + } + + func withPermit(_ operation: () async throws -> T) async throws -> T { + await self.acquire() + defer { self.release() } + if Task.isCancelled { throw CancellationError() } + return try await operation() + } + + private func acquire() async { + if self.available > 0 { + self.available -= 1 + return + } + let id = UUID() + await withCheckedContinuation { cont in + self.waitQueue.append(id) + self.waiters[id] = cont + } + } + + private func release() { + if let id = self.waitQueue.first { + self.waitQueue.removeFirst() + if let cont = self.waiters.removeValue(forKey: id) { + cont.resume() + } + return + } + self.available = min(self.available + 1, self.maxConcurrent) + } +} + +#if DEBUG +extension SessionPreviewCache { + func _testSet( + snapshot: SessionMenuPreviewSnapshot, + for sessionKey: String, + updatedAt: Date = Date()) + { + self.entries[sessionKey] = CacheEntry(snapshot: snapshot, updatedAt: updatedAt) + } + + func _testReset() { + self.entries = [:] + } +} +#endif + +struct SessionMenuPreviewSnapshot: Sendable { + let items: [SessionPreviewItem] + let status: SessionMenuPreviewView.LoadStatus +} + +struct SessionMenuPreviewView: View { + let width: CGFloat + let maxLines: Int + let title: String + let items: [SessionPreviewItem] + let status: LoadStatus + + @Environment(\.menuItemHighlighted) private var isHighlighted + + enum LoadStatus: Equatable { + case loading + case ready + case empty + case error(String) + } + + private var primaryColor: Color { + if self.isHighlighted { + return Color(nsColor: .selectedMenuItemTextColor) + } + return Color(nsColor: .labelColor) + } + + private var secondaryColor: Color { + if self.isHighlighted { + return Color(nsColor: .selectedMenuItemTextColor).opacity(0.85) + } + return Color(nsColor: .secondaryLabelColor) + } + + var body: some View { + VStack(alignment: .leading, spacing: 8) { + HStack(alignment: .firstTextBaseline, spacing: 4) { + Text(self.title) + .font(.caption.weight(.semibold)) + .foregroundStyle(self.secondaryColor) + Spacer(minLength: 8) + } + + switch self.status { + case .loading: + self.placeholder("Loading preview…") + case .empty: + self.placeholder("No recent messages") + case let .error(message): + self.placeholder(message) + case .ready: + if self.items.isEmpty { + self.placeholder("No recent messages") + } else { + VStack(alignment: .leading, spacing: 6) { + ForEach(self.items) { item in + self.previewRow(item) + } + } + } + } + } + .padding(.vertical, 6) + .padding(.leading, 16) + .padding(.trailing, 11) + .frame(width: max(1, self.width), alignment: .leading) + } + + @ViewBuilder + private func previewRow(_ item: SessionPreviewItem) -> some View { + HStack(alignment: .top, spacing: 4) { + Text(item.role.label) + .font(.caption2.monospacedDigit()) + .foregroundStyle(self.roleColor(item.role)) + .frame(width: 50, alignment: .leading) + + Text(item.text) + .font(.caption) + .foregroundStyle(self.primaryColor) + .multilineTextAlignment(.leading) + .lineLimit(self.maxLines) + .truncationMode(.tail) + .fixedSize(horizontal: false, vertical: true) + } + } + + private func roleColor(_ role: PreviewRole) -> Color { + if self.isHighlighted { return Color(nsColor: .selectedMenuItemTextColor).opacity(0.9) } + switch role { + case .user: return .accentColor + case .assistant: return .secondary + case .tool: return .orange + case .system: return .gray + case .other: return .secondary + } + } + + @ViewBuilder + private func placeholder(_ text: String) -> some View { + Text(text) + .font(.caption) + .foregroundStyle(self.primaryColor) + } +} + +enum SessionMenuPreviewLoader { + private static let logger = Logger(subsystem: "ai.openclaw", category: "SessionPreview") + private static let previewTimeoutSeconds: Double = 4 + private static let cacheMaxAgeSeconds: TimeInterval = 30 + private static let previewMaxChars = 240 + + private struct PreviewTimeoutError: LocalizedError { + var errorDescription: String? { "preview timeout" } + } + + static func prewarm(sessionKeys: [String], maxItems: Int) async { + let keys = self.uniqueKeys(sessionKeys) + guard !keys.isEmpty else { return } + do { + let payload = try await self.requestPreview(keys: keys, maxItems: maxItems) + await self.cache(payload: payload, maxItems: maxItems) + } catch { + if self.isUnknownMethodError(error) { return } + let errorDescription = String(describing: error) + Self.logger.debug( + "Session preview prewarm failed count=\(keys.count, privacy: .public) " + + "error=\(errorDescription, privacy: .public)") + } + } + + static func load(sessionKey: String, maxItems: Int) async -> SessionMenuPreviewSnapshot { + if let cached = await SessionPreviewCache.shared.cachedSnapshot( + for: sessionKey, + maxAge: cacheMaxAgeSeconds) + { + return cached + } + + do { + let snapshot = try await self.fetchSnapshot(sessionKey: sessionKey, maxItems: maxItems) + await SessionPreviewCache.shared.store(snapshot: snapshot, for: sessionKey) + return snapshot + } catch is CancellationError { + return SessionMenuPreviewSnapshot(items: [], status: .loading) + } catch { + if let fallback = await SessionPreviewCache.shared.lastSnapshot(for: sessionKey) { + return fallback + } + let errorDescription = String(describing: error) + Self.logger.warning( + "Session preview failed session=\(sessionKey, privacy: .public) " + + "error=\(errorDescription, privacy: .public)") + return SessionMenuPreviewSnapshot(items: [], status: .error("Preview unavailable")) + } + } + + private static func fetchSnapshot(sessionKey: String, maxItems: Int) async throws -> SessionMenuPreviewSnapshot { + do { + let payload = try await self.requestPreview(keys: [sessionKey], maxItems: maxItems) + if let entry = payload.previews.first(where: { $0.key == sessionKey }) ?? payload.previews.first { + return self.snapshot(from: entry, maxItems: maxItems) + } + return SessionMenuPreviewSnapshot(items: [], status: .error("Preview unavailable")) + } catch { + if self.isUnknownMethodError(error) { + return try await self.fetchHistorySnapshot(sessionKey: sessionKey, maxItems: maxItems) + } + throw error + } + } + + private static func requestPreview( + keys: [String], + maxItems: Int) async throws -> OpenClawSessionsPreviewPayload + { + let boundedItems = self.normalizeMaxItems(maxItems) + let timeoutMs = Int(self.previewTimeoutSeconds * 1000) + return try await SessionPreviewLimiter.shared.withPermit { + try await AsyncTimeout.withTimeout( + seconds: self.previewTimeoutSeconds, + onTimeout: { PreviewTimeoutError() }, + operation: { + try await GatewayConnection.shared.sessionsPreview( + keys: keys, + limit: boundedItems, + maxChars: self.previewMaxChars, + timeoutMs: timeoutMs) + }) + } + } + + private static func fetchHistorySnapshot( + sessionKey: String, + maxItems: Int) async throws -> SessionMenuPreviewSnapshot + { + let timeoutMs = Int(self.previewTimeoutSeconds * 1000) + let payload = try await SessionPreviewLimiter.shared.withPermit { + try await AsyncTimeout.withTimeout( + seconds: self.previewTimeoutSeconds, + onTimeout: { PreviewTimeoutError() }, + operation: { + try await GatewayConnection.shared.chatHistory( + sessionKey: sessionKey, + limit: self.previewLimit(for: maxItems), + timeoutMs: timeoutMs) + }) + } + let built = Self.previewItems(from: payload, maxItems: maxItems) + return Self.snapshot(from: built) + } + + private static func snapshot(from items: [SessionPreviewItem]) -> SessionMenuPreviewSnapshot { + SessionMenuPreviewSnapshot(items: items, status: items.isEmpty ? .empty : .ready) + } + + private static func snapshot( + from entry: OpenClawSessionPreviewEntry, + maxItems: Int) -> SessionMenuPreviewSnapshot + { + let items = self.previewItems(from: entry, maxItems: maxItems) + let normalized = entry.status.trimmingCharacters(in: .whitespacesAndNewlines).lowercased() + switch normalized { + case "ok": + return SessionMenuPreviewSnapshot(items: items, status: items.isEmpty ? .empty : .ready) + case "empty": + return SessionMenuPreviewSnapshot(items: items, status: .empty) + case "missing": + return SessionMenuPreviewSnapshot(items: items, status: .error("Session missing")) + default: + return SessionMenuPreviewSnapshot(items: items, status: .error("Preview unavailable")) + } + } + + private static func cache(payload: OpenClawSessionsPreviewPayload, maxItems: Int) async { + for entry in payload.previews { + let snapshot = self.snapshot(from: entry, maxItems: maxItems) + await SessionPreviewCache.shared.store(snapshot: snapshot, for: entry.key) + } + } + + private static func previewLimit(for maxItems: Int) -> Int { + let boundedItems = self.normalizeMaxItems(maxItems) + return min(max(boundedItems * 3, 20), 120) + } + + private static func normalizeMaxItems(_ maxItems: Int) -> Int { + max(1, min(maxItems, 50)) + } + + private static func previewItems( + from entry: OpenClawSessionPreviewEntry, + maxItems: Int) -> [SessionPreviewItem] + { + let boundedItems = self.normalizeMaxItems(maxItems) + let built: [SessionPreviewItem] = entry.items.enumerated().compactMap { index, item in + let text = item.text.trimmingCharacters(in: .whitespacesAndNewlines) + guard !text.isEmpty else { return nil } + let role = self.previewRoleFromRaw(item.role) + return SessionPreviewItem(id: "\(entry.key)-\(index)", role: role, text: text) + } + + let trimmed = built.suffix(boundedItems) + return Array(trimmed.reversed()) + } + + private static func previewItems( + from payload: OpenClawChatHistoryPayload, + maxItems: Int) -> [SessionPreviewItem] + { + let boundedItems = self.normalizeMaxItems(maxItems) + let raw: [OpenClawKit.AnyCodable] = payload.messages ?? [] + let messages = self.decodeMessages(raw) + let built = messages.compactMap { message -> SessionPreviewItem? in + guard let text = self.previewText(for: message) else { return nil } + let isTool = self.isToolCall(message) + let role = self.previewRole(message.role, isTool: isTool) + let id = "\(message.timestamp ?? 0)-\(UUID().uuidString)" + return SessionPreviewItem(id: id, role: role, text: text) + } + + let trimmed = built.suffix(boundedItems) + return Array(trimmed.reversed()) + } + + private static func decodeMessages(_ raw: [OpenClawKit.AnyCodable]) -> [OpenClawChatMessage] { + raw.compactMap { item in + guard let data = try? JSONEncoder().encode(item) else { return nil } + return try? JSONDecoder().decode(OpenClawChatMessage.self, from: data) + } + } + + private static func previewRole(_ raw: String, isTool: Bool) -> PreviewRole { + if isTool { return .tool } + return self.previewRoleFromRaw(raw) + } + + private static func previewRoleFromRaw(_ raw: String) -> PreviewRole { + switch raw.lowercased() { + case "user": .user + case "assistant": .assistant + case "system": .system + case "tool": .tool + default: .other + } + } + + private static func previewText(for message: OpenClawChatMessage) -> String? { + let text = message.content.compactMap(\.text).joined(separator: "\n") + .trimmingCharacters(in: .whitespacesAndNewlines) + if !text.isEmpty { return text } + + let toolNames = self.toolNames(for: message) + if !toolNames.isEmpty { + let shown = toolNames.prefix(2) + let overflow = toolNames.count - shown.count + var label = "call \(shown.joined(separator: ", "))" + if overflow > 0 { label += " +\(overflow)" } + return label + } + + if let media = self.mediaSummary(for: message) { + return media + } + + return nil + } + + private static func isToolCall(_ message: OpenClawChatMessage) -> Bool { + if message.toolName?.nonEmpty != nil { return true } + return message.content.contains { $0.name?.nonEmpty != nil || $0.type?.lowercased() == "toolcall" } + } + + private static func toolNames(for message: OpenClawChatMessage) -> [String] { + var names: [String] = [] + for content in message.content { + if let name = content.name?.nonEmpty { + names.append(name) + } + } + if let toolName = message.toolName?.nonEmpty { + names.append(toolName) + } + return Self.dedupePreservingOrder(names) + } + + private static func mediaSummary(for message: OpenClawChatMessage) -> String? { + let types = message.content.compactMap { content -> String? in + let raw = content.type?.trimmingCharacters(in: .whitespacesAndNewlines).lowercased() + guard let raw, !raw.isEmpty else { return nil } + if raw == "text" || raw == "toolcall" { return nil } + return raw + } + guard let first = types.first else { return nil } + return "[\(first)]" + } + + private static func dedupePreservingOrder(_ values: [String]) -> [String] { + var seen = Set() + var result: [String] = [] + for value in values where !seen.contains(value) { + seen.insert(value) + result.append(value) + } + return result + } + + private static func uniqueKeys(_ keys: [String]) -> [String] { + let trimmed = keys.map { $0.trimmingCharacters(in: .whitespacesAndNewlines) } + return self.dedupePreservingOrder(trimmed.filter { !$0.isEmpty }) + } + + private static func isUnknownMethodError(_ error: Error) -> Bool { + guard let response = error as? GatewayResponseError else { return false } + guard response.code == ErrorCode.invalidRequest.rawValue else { return false } + let message = response.message.lowercased() + return message.contains("unknown method") + } +} diff --git a/apps/macos/Sources/OpenClaw/SessionsSettings.swift b/apps/macos/Sources/OpenClaw/SessionsSettings.swift new file mode 100644 index 0000000000000000000000000000000000000000..4a2a0e81e02970be1eb329830767f11fa83af0dc --- /dev/null +++ b/apps/macos/Sources/OpenClaw/SessionsSettings.swift @@ -0,0 +1,213 @@ +import AppKit +import SwiftUI + +@MainActor +struct SessionsSettings: View { + private let isPreview: Bool + @State private var rows: [SessionRow] + @State private var errorMessage: String? + @State private var loading = false + @State private var hasLoaded = false + + init(rows: [SessionRow]? = nil, isPreview: Bool = ProcessInfo.processInfo.isPreview) { + self._rows = State(initialValue: rows ?? []) + self.isPreview = isPreview + if isPreview { + self._hasLoaded = State(initialValue: true) + } + } + + var body: some View { + VStack(alignment: .leading, spacing: 14) { + self.header + self.content + Spacer() + } + .frame(maxWidth: .infinity, alignment: .leading) + .padding(.horizontal, 12) + .task { + guard !self.hasLoaded else { return } + guard !self.isPreview else { return } + self.hasLoaded = true + await self.refresh() + } + } + + private var header: some View { + HStack(alignment: .top, spacing: 12) { + VStack(alignment: .leading, spacing: 4) { + Text("Sessions") + .font(.headline) + Text("Peek at the stored conversation buckets the CLI reuses for context and rate limits.") + .font(.footnote) + .foregroundStyle(.secondary) + .fixedSize(horizontal: false, vertical: true) + } + Spacer() + if self.loading { + ProgressView() + } else { + Button { + Task { await self.refresh() } + } label: { + Label("Refresh", systemImage: "arrow.clockwise") + } + .buttonStyle(.bordered) + .help("Refresh") + } + } + } + + private var content: some View { + Group { + if self.rows.isEmpty, self.errorMessage == nil { + Text("No sessions yet. They appear after the first inbound message or heartbeat.") + .font(.footnote) + .foregroundStyle(.secondary) + .padding(.top, 6) + } else { + List(self.rows) { row in + self.sessionRow(row) + } + .listStyle(.inset) + .overlay(alignment: .topLeading) { + if let errorMessage { + Text(errorMessage) + .font(.footnote) + .foregroundStyle(.red) + .padding(.leading, 4) + .padding(.top, 4) + } + } + // The view already applies horizontal padding; keep the list aligned with the text above. + .padding(.horizontal, -12) + } + } + } + + @ViewBuilder + private func sessionRow(_ row: SessionRow) -> some View { + VStack(alignment: .leading, spacing: 6) { + HStack(alignment: .firstTextBaseline, spacing: 8) { + Text(row.label) + .font(.subheadline.bold()) + .lineLimit(1) + .truncationMode(.middle) + Spacer() + Text(row.ageText) + .font(.caption) + .foregroundStyle(.secondary) + } + + HStack(spacing: 6) { + if row.kind != .direct { + SessionKindBadge(kind: row.kind) + } + if !row.flagLabels.isEmpty { + ForEach(row.flagLabels, id: \.self) { flag in + Badge(text: flag) + } + } + } + + VStack(alignment: .leading, spacing: 6) { + HStack(spacing: 8) { + Text("Context") + .font(.caption.weight(.semibold)) + .foregroundStyle(.secondary) + Spacer() + Text(row.tokens.contextSummaryShort) + .font(.caption.monospacedDigit()) + .foregroundStyle(.secondary) + } + ContextUsageBar( + usedTokens: row.tokens.total, + contextTokens: row.tokens.contextTokens, + width: nil) + } + + HStack(spacing: 10) { + if let model = row.model, !model.isEmpty { + self.label(icon: "cpu", text: model) + } + self.label(icon: "arrow.down.left", text: "\(row.tokens.input) in") + self.label(icon: "arrow.up.right", text: "\(row.tokens.output) out") + if let sessionId = row.sessionId, !sessionId.isEmpty { + HStack(spacing: 4) { + Image(systemName: "number").foregroundStyle(.secondary).font(.caption) + Text(sessionId) + .font(.footnote.monospaced()) + .foregroundStyle(.secondary) + .lineLimit(1) + .truncationMode(.middle) + } + .help(sessionId) + } + } + } + .padding(.vertical, 6) + } + + private func label(icon: String, text: String) -> some View { + HStack(spacing: 4) { + Image(systemName: icon).foregroundStyle(.secondary).font(.caption) + Text(text) + } + .font(.footnote) + .foregroundStyle(.secondary) + } + + private func refresh() async { + guard !self.loading else { return } + guard !self.isPreview else { return } + self.loading = true + self.errorMessage = nil + + do { + let snapshot = try await SessionLoader.loadSnapshot() + self.rows = snapshot.rows + } catch { + self.rows = [] + self.errorMessage = (error as? LocalizedError)?.errorDescription ?? error.localizedDescription + } + + self.loading = false + } +} + +private struct SessionKindBadge: View { + let kind: SessionKind + + var body: some View { + Text(self.kind.label) + .font(.caption2.weight(.bold)) + .padding(.horizontal, 7) + .padding(.vertical, 4) + .foregroundStyle(self.kind.tint) + .background(self.kind.tint.opacity(0.15)) + .clipShape(Capsule()) + } +} + +private struct Badge: View { + let text: String + + var body: some View { + Text(self.text) + .font(.caption2.weight(.semibold)) + .padding(.horizontal, 6) + .padding(.vertical, 3) + .foregroundStyle(.secondary) + .background(Color.secondary.opacity(0.12)) + .clipShape(Capsule()) + } +} + +#if DEBUG +struct SessionsSettings_Previews: PreviewProvider { + static var previews: some View { + SessionsSettings(rows: SessionRow.previewRows, isPreview: true) + .frame(width: SettingsTab.windowWidth, height: SettingsTab.windowHeight) + } +} +#endif diff --git a/apps/macos/Sources/OpenClaw/SettingsComponents.swift b/apps/macos/Sources/OpenClaw/SettingsComponents.swift new file mode 100644 index 0000000000000000000000000000000000000000..f826fd4e52c942a4bae897cd03dd06d0b27daccf --- /dev/null +++ b/apps/macos/Sources/OpenClaw/SettingsComponents.swift @@ -0,0 +1,24 @@ +import SwiftUI + +struct SettingsToggleRow: View { + let title: String + let subtitle: String? + @Binding var binding: Bool + + var body: some View { + VStack(alignment: .leading, spacing: 6) { + Toggle(isOn: self.$binding) { + Text(self.title) + .font(.body) + } + .toggleStyle(.checkbox) + + if let subtitle, !subtitle.isEmpty { + Text(subtitle) + .font(.footnote) + .foregroundStyle(.tertiary) + .fixedSize(horizontal: false, vertical: true) + } + } + } +} diff --git a/apps/macos/Sources/OpenClaw/SettingsRootView.swift b/apps/macos/Sources/OpenClaw/SettingsRootView.swift new file mode 100644 index 0000000000000000000000000000000000000000..016e2f3d1c7d7330006e172536729b6976774196 --- /dev/null +++ b/apps/macos/Sources/OpenClaw/SettingsRootView.swift @@ -0,0 +1,243 @@ +import Observation +import SwiftUI + +struct SettingsRootView: View { + @Bindable var state: AppState + private let permissionMonitor = PermissionMonitor.shared + @State private var monitoringPermissions = false + @State private var selectedTab: SettingsTab = .general + @State private var snapshotPaths: (configPath: String?, stateDir: String?) = (nil, nil) + let updater: UpdaterProviding? + private let isPreview = ProcessInfo.processInfo.isPreview + private let isNixMode = ProcessInfo.processInfo.isNixMode + + init(state: AppState, updater: UpdaterProviding?, initialTab: SettingsTab? = nil) { + self.state = state + self.updater = updater + self._selectedTab = State(initialValue: initialTab ?? .general) + } + + var body: some View { + VStack(alignment: .leading, spacing: 12) { + if self.isNixMode { + self.nixManagedBanner + } + TabView(selection: self.$selectedTab) { + GeneralSettings(state: self.state) + .tabItem { Label("General", systemImage: "gearshape") } + .tag(SettingsTab.general) + + ChannelsSettings() + .tabItem { Label("Channels", systemImage: "link") } + .tag(SettingsTab.channels) + + VoiceWakeSettings(state: self.state, isActive: self.selectedTab == .voiceWake) + .tabItem { Label("Voice Wake", systemImage: "waveform.circle") } + .tag(SettingsTab.voiceWake) + + ConfigSettings() + .tabItem { Label("Config", systemImage: "slider.horizontal.3") } + .tag(SettingsTab.config) + + InstancesSettings() + .tabItem { Label("Instances", systemImage: "network") } + .tag(SettingsTab.instances) + + SessionsSettings() + .tabItem { Label("Sessions", systemImage: "clock.arrow.circlepath") } + .tag(SettingsTab.sessions) + + CronSettings() + .tabItem { Label("Cron", systemImage: "calendar") } + .tag(SettingsTab.cron) + + SkillsSettings(state: self.state) + .tabItem { Label("Skills", systemImage: "sparkles") } + .tag(SettingsTab.skills) + + PermissionsSettings( + status: self.permissionMonitor.status, + refresh: self.refreshPerms, + showOnboarding: { DebugActions.restartOnboarding() }) + .tabItem { Label("Permissions", systemImage: "lock.shield") } + .tag(SettingsTab.permissions) + + if self.state.debugPaneEnabled { + DebugSettings(state: self.state) + .tabItem { Label("Debug", systemImage: "ant") } + .tag(SettingsTab.debug) + } + + AboutSettings(updater: self.updater) + .tabItem { Label("About", systemImage: "info.circle") } + .tag(SettingsTab.about) + } + } + .padding(.horizontal, 28) + .padding(.vertical, 22) + .frame(width: SettingsTab.windowWidth, height: SettingsTab.windowHeight, alignment: .topLeading) + .frame(maxWidth: .infinity, maxHeight: .infinity, alignment: .topLeading) + .onReceive(NotificationCenter.default.publisher(for: .openclawSelectSettingsTab)) { note in + if let tab = note.object as? SettingsTab { + withAnimation(.spring(response: 0.32, dampingFraction: 0.85)) { + self.selectedTab = tab + } + } + } + .onAppear { + if let pending = SettingsTabRouter.consumePending() { + self.selectedTab = self.validTab(for: pending) + } + self.updatePermissionMonitoring(for: self.selectedTab) + } + .onChange(of: self.state.debugPaneEnabled) { _, enabled in + if !enabled, self.selectedTab == .debug { + self.selectedTab = .general + } + } + .onChange(of: self.selectedTab) { _, newValue in + self.updatePermissionMonitoring(for: newValue) + } + .onDisappear { self.stopPermissionMonitoring() } + .task { + guard !self.isPreview else { return } + await self.refreshPerms() + } + .task(id: self.state.connectionMode) { + guard !self.isPreview else { return } + await self.refreshSnapshotPaths() + } + } + + private var nixManagedBanner: some View { + // Prefer gateway-resolved paths; fall back to local env defaults if disconnected. + let configPath = self.snapshotPaths.configPath ?? OpenClawPaths.configURL.path + let stateDir = self.snapshotPaths.stateDir ?? OpenClawPaths.stateDirURL.path + + return VStack(alignment: .leading, spacing: 6) { + HStack(spacing: 8) { + Image(systemName: "gearshape.2.fill") + .foregroundStyle(.secondary) + Text("Managed by Nix") + .font(.callout.weight(.semibold)) + .foregroundStyle(.secondary) + } + + VStack(alignment: .leading, spacing: 2) { + Text("Config: \(configPath)") + Text("State: \(stateDir)") + } + .font(.caption.monospaced()) + .foregroundStyle(.secondary) + .textSelection(.enabled) + .lineLimit(1) + .truncationMode(.middle) + } + .padding(.vertical, 8) + .padding(.horizontal, 10) + .background(Color.gray.opacity(0.12)) + .cornerRadius(10) + } + + private func validTab(for requested: SettingsTab) -> SettingsTab { + if requested == .debug, !self.state.debugPaneEnabled { return .general } + return requested + } + + @MainActor + private func refreshSnapshotPaths() async { + let paths = await GatewayConnection.shared.snapshotPaths() + self.snapshotPaths = paths + } + + @MainActor + private func refreshPerms() async { + guard !self.isPreview else { return } + await self.permissionMonitor.refreshNow() + } + + private func updatePermissionMonitoring(for tab: SettingsTab) { + guard !self.isPreview else { return } + let shouldMonitor = tab == .permissions + if shouldMonitor, !self.monitoringPermissions { + self.monitoringPermissions = true + PermissionMonitor.shared.register() + } else if !shouldMonitor, self.monitoringPermissions { + self.monitoringPermissions = false + PermissionMonitor.shared.unregister() + } + } + + private func stopPermissionMonitoring() { + guard self.monitoringPermissions else { return } + self.monitoringPermissions = false + PermissionMonitor.shared.unregister() + } +} + +enum SettingsTab: CaseIterable { + case general, channels, skills, sessions, cron, config, instances, voiceWake, permissions, debug, about + static let windowWidth: CGFloat = 824 // wider + static let windowHeight: CGFloat = 790 // +10% (more room) + var title: String { + switch self { + case .general: "General" + case .channels: "Channels" + case .skills: "Skills" + case .sessions: "Sessions" + case .cron: "Cron" + case .config: "Config" + case .instances: "Instances" + case .voiceWake: "Voice Wake" + case .permissions: "Permissions" + case .debug: "Debug" + case .about: "About" + } + } + + var systemImage: String { + switch self { + case .general: "gearshape" + case .channels: "link" + case .skills: "sparkles" + case .sessions: "clock.arrow.circlepath" + case .cron: "calendar" + case .config: "slider.horizontal.3" + case .instances: "network" + case .voiceWake: "waveform.circle" + case .permissions: "lock.shield" + case .debug: "ant" + case .about: "info.circle" + } + } +} + +@MainActor +enum SettingsTabRouter { + private static var pending: SettingsTab? + + static func request(_ tab: SettingsTab) { + self.pending = tab + } + + static func consumePending() -> SettingsTab? { + defer { self.pending = nil } + return self.pending + } +} + +extension Notification.Name { + static let openclawSelectSettingsTab = Notification.Name("openclawSelectSettingsTab") +} + +#if DEBUG +struct SettingsRootView_Previews: PreviewProvider { + static var previews: some View { + ForEach(SettingsTab.allCases, id: \.self) { tab in + SettingsRootView(state: .preview, updater: DisabledUpdaterController(), initialTab: tab) + .previewDisplayName(tab.title) + .frame(width: SettingsTab.windowWidth, height: SettingsTab.windowHeight) + } + } +} +#endif diff --git a/apps/macos/Sources/OpenClaw/SettingsWindowOpener.swift b/apps/macos/Sources/OpenClaw/SettingsWindowOpener.swift new file mode 100644 index 0000000000000000000000000000000000000000..9cc1647b6f5302a2845102db977af03b763b50c9 --- /dev/null +++ b/apps/macos/Sources/OpenClaw/SettingsWindowOpener.swift @@ -0,0 +1,36 @@ +import AppKit +import SwiftUI + +@objc +private protocol SettingsWindowMenuActions { + @objc(showSettingsWindow:) + optional func showSettingsWindow(_ sender: Any?) + + @objc(showPreferencesWindow:) + optional func showPreferencesWindow(_ sender: Any?) +} + +@MainActor +final class SettingsWindowOpener { + static let shared = SettingsWindowOpener() + + private var openSettingsAction: OpenSettingsAction? + + func register(openSettings: OpenSettingsAction) { + self.openSettingsAction = openSettings + } + + func open() { + NSApp.activate(ignoringOtherApps: true) + if let openSettingsAction { + openSettingsAction() + return + } + + // Fallback path: mimic the built-in Settings menu item action. + let didOpen = NSApp.sendAction(#selector(SettingsWindowMenuActions.showSettingsWindow(_:)), to: nil, from: nil) + if !didOpen { + _ = NSApp.sendAction(#selector(SettingsWindowMenuActions.showPreferencesWindow(_:)), to: nil, from: nil) + } + } +} diff --git a/apps/macos/Sources/OpenClaw/ShellExecutor.swift b/apps/macos/Sources/OpenClaw/ShellExecutor.swift new file mode 100644 index 0000000000000000000000000000000000000000..9633f0f8da0a608cdce07796b10ab4ee8596044b --- /dev/null +++ b/apps/macos/Sources/OpenClaw/ShellExecutor.swift @@ -0,0 +1,102 @@ +import OpenClawIPC +import Foundation + +enum ShellExecutor { + struct ShellResult { + var stdout: String + var stderr: String + var exitCode: Int? + var timedOut: Bool + var success: Bool + var errorMessage: String? + } + + static func runDetailed( + command: [String], + cwd: String?, + env: [String: String]?, + timeout: Double?) async -> ShellResult + { + guard !command.isEmpty else { + return ShellResult( + stdout: "", + stderr: "", + exitCode: nil, + timedOut: false, + success: false, + errorMessage: "empty command") + } + + let process = Process() + process.executableURL = URL(fileURLWithPath: "/usr/bin/env") + process.arguments = command + if let cwd { process.currentDirectoryURL = URL(fileURLWithPath: cwd) } + if let env { process.environment = env } + + let stdoutPipe = Pipe() + let stderrPipe = Pipe() + process.standardOutput = stdoutPipe + process.standardError = stderrPipe + + do { + try process.run() + } catch { + return ShellResult( + stdout: "", + stderr: "", + exitCode: nil, + timedOut: false, + success: false, + errorMessage: "failed to start: \(error.localizedDescription)") + } + + let outTask = Task { stdoutPipe.fileHandleForReading.readToEndSafely() } + let errTask = Task { stderrPipe.fileHandleForReading.readToEndSafely() } + + let waitTask = Task { () -> ShellResult in + process.waitUntilExit() + let out = await outTask.value + let err = await errTask.value + let status = Int(process.terminationStatus) + return ShellResult( + stdout: String(bytes: out, encoding: .utf8) ?? "", + stderr: String(bytes: err, encoding: .utf8) ?? "", + exitCode: status, + timedOut: false, + success: status == 0, + errorMessage: status == 0 ? nil : "exit \(status)") + } + + if let timeout, timeout > 0 { + let nanos = UInt64(timeout * 1_000_000_000) + let result = await withTaskGroup(of: ShellResult.self) { group in + group.addTask { await waitTask.value } + group.addTask { + try? await Task.sleep(nanoseconds: nanos) + if process.isRunning { process.terminate() } + _ = await waitTask.value // drain pipes after termination + return ShellResult( + stdout: "", + stderr: "", + exitCode: nil, + timedOut: true, + success: false, + errorMessage: "timeout") + } + let first = await group.next()! + group.cancelAll() + return first + } + return result + } + + return await waitTask.value + } + + static func run(command: [String], cwd: String?, env: [String: String]?, timeout: Double?) async -> Response { + let result = await self.runDetailed(command: command, cwd: cwd, env: env, timeout: timeout) + let combined = result.stdout.isEmpty ? result.stderr : result.stdout + let payload = combined.isEmpty ? nil : Data(combined.utf8) + return Response(ok: result.success, message: result.errorMessage, payload: payload) + } +} diff --git a/apps/macos/Sources/OpenClaw/SkillsModels.swift b/apps/macos/Sources/OpenClaw/SkillsModels.swift new file mode 100644 index 0000000000000000000000000000000000000000..1fb40d99f15970b5d5760fcf6c1408bdd53c9a84 --- /dev/null +++ b/apps/macos/Sources/OpenClaw/SkillsModels.swift @@ -0,0 +1,70 @@ +import OpenClawProtocol +import Foundation + +struct SkillsStatusReport: Codable { + let workspaceDir: String + let managedSkillsDir: String + let skills: [SkillStatus] +} + +struct SkillStatus: Codable, Identifiable { + let name: String + let description: String + let source: String + let filePath: String + let baseDir: String + let skillKey: String + let primaryEnv: String? + let emoji: String? + let homepage: String? + let always: Bool + let disabled: Bool + let eligible: Bool + let requirements: SkillRequirements + let missing: SkillMissing + let configChecks: [SkillStatusConfigCheck] + let install: [SkillInstallOption] + + var id: String { self.name } +} + +struct SkillRequirements: Codable { + let bins: [String] + let env: [String] + let config: [String] +} + +struct SkillMissing: Codable { + let bins: [String] + let env: [String] + let config: [String] +} + +struct SkillStatusConfigCheck: Codable, Identifiable { + let path: String + let value: AnyCodable? + let satisfied: Bool + + var id: String { self.path } +} + +struct SkillInstallOption: Codable, Identifiable { + let id: String + let kind: String + let label: String + let bins: [String] +} + +struct SkillInstallResult: Codable { + let ok: Bool + let message: String + let stdout: String? + let stderr: String? + let code: Int? +} + +struct SkillUpdateResult: Codable { + let ok: Bool + let skillKey: String + let config: [String: AnyCodable]? +} diff --git a/apps/macos/Sources/OpenClaw/SkillsSettings.swift b/apps/macos/Sources/OpenClaw/SkillsSettings.swift new file mode 100644 index 0000000000000000000000000000000000000000..83aaa66c55db4e849b77a1401a01e60d369e528e --- /dev/null +++ b/apps/macos/Sources/OpenClaw/SkillsSettings.swift @@ -0,0 +1,628 @@ +import OpenClawProtocol +import Observation +import SwiftUI + +struct SkillsSettings: View { + @Bindable var state: AppState + @State private var model = SkillsSettingsModel() + @State private var envEditor: EnvEditorState? + @State private var filter: SkillsFilter = .all + + init(state: AppState = AppStateStore.shared, model: SkillsSettingsModel = SkillsSettingsModel()) { + self.state = state + self._model = State(initialValue: model) + } + + var body: some View { + VStack(alignment: .leading, spacing: 12) { + self.header + self.statusBanner + self.skillsList + Spacer(minLength: 0) + } + .task { await self.model.refresh() } + .sheet(item: self.$envEditor) { editor in + EnvEditorView(editor: editor) { value in + Task { + await self.model.updateEnv( + skillKey: editor.skillKey, + envKey: editor.envKey, + value: value, + isPrimary: editor.isPrimary) + } + } + } + } + + private var header: some View { + HStack { + VStack(alignment: .leading, spacing: 4) { + Text("Skills") + .font(.headline) + Text("Skills are enabled when requirements are met (binaries, env, config).") + .font(.footnote) + .foregroundStyle(.secondary) + } + Spacer() + if self.model.isLoading { + ProgressView() + } else { + Button { + Task { await self.model.refresh() } + } label: { + Label("Refresh", systemImage: "arrow.clockwise") + } + .buttonStyle(.bordered) + .help("Refresh") + } + self.headerFilter + } + } + + @ViewBuilder + private var statusBanner: some View { + if let error = self.model.error { + Text(error) + .font(.footnote) + .foregroundStyle(.orange) + } else if let message = self.model.statusMessage { + Text(message) + .font(.footnote) + .foregroundStyle(.secondary) + } + } + + @ViewBuilder + private var skillsList: some View { + if self.model.skills.isEmpty { + Text("No skills reported yet.") + .foregroundStyle(.secondary) + } else { + List { + ForEach(self.filteredSkills) { skill in + SkillRow( + skill: skill, + isBusy: self.model.isBusy(skill: skill), + connectionMode: self.state.connectionMode, + onToggleEnabled: { enabled in + Task { await self.model.setEnabled(skillKey: skill.skillKey, enabled: enabled) } + }, + onInstall: { option, target in + Task { await self.model.install(skill: skill, option: option, target: target) } + }, + onSetEnv: { envKey, isPrimary in + self.envEditor = EnvEditorState( + skillKey: skill.skillKey, + skillName: skill.name, + envKey: envKey, + isPrimary: isPrimary) + }) + } + if !self.model.skills.isEmpty, self.filteredSkills.isEmpty { + Text("No skills match this filter.") + .font(.callout) + .foregroundStyle(.secondary) + } + } + .listStyle(.inset) + } + } + + private var headerFilter: some View { + Picker("Filter", selection: self.$filter) { + ForEach(SkillsFilter.allCases) { filter in + Text(filter.title) + .tag(filter) + } + } + .labelsHidden() + .pickerStyle(.menu) + .frame(width: 160, alignment: .trailing) + } + + private var filteredSkills: [SkillStatus] { + self.model.skills.filter { skill in + switch self.filter { + case .all: + true + case .ready: + !skill.disabled && skill.eligible + case .needsSetup: + !skill.disabled && !skill.eligible + case .disabled: + skill.disabled + } + } + } +} + +private enum SkillsFilter: String, CaseIterable, Identifiable { + case all + case ready + case needsSetup + case disabled + + var id: String { self.rawValue } + + var title: String { + switch self { + case .all: + "All" + case .ready: + "Ready" + case .needsSetup: + "Needs Setup" + case .disabled: + "Disabled" + } + } +} + +private enum InstallTarget: String, CaseIterable { + case gateway + case local +} + +private struct SkillRow: View { + let skill: SkillStatus + let isBusy: Bool + let connectionMode: AppState.ConnectionMode + let onToggleEnabled: (Bool) -> Void + let onInstall: (SkillInstallOption, InstallTarget) -> Void + let onSetEnv: (String, Bool) -> Void + + private var missingBins: [String] { self.skill.missing.bins } + private var missingEnv: [String] { self.skill.missing.env } + private var missingConfig: [String] { self.skill.missing.config } + + init( + skill: SkillStatus, + isBusy: Bool, + connectionMode: AppState.ConnectionMode, + onToggleEnabled: @escaping (Bool) -> Void, + onInstall: @escaping (SkillInstallOption, InstallTarget) -> Void, + onSetEnv: @escaping (String, Bool) -> Void) + { + self.skill = skill + self.isBusy = isBusy + self.connectionMode = connectionMode + self.onToggleEnabled = onToggleEnabled + self.onInstall = onInstall + self.onSetEnv = onSetEnv + } + + var body: some View { + HStack(alignment: .top, spacing: 12) { + Text(self.skill.emoji ?? "✨") + .font(.title2) + + VStack(alignment: .leading, spacing: 6) { + Text(self.skill.name) + .font(.headline) + Text(self.skill.description) + .font(.subheadline) + .foregroundStyle(.secondary) + .fixedSize(horizontal: false, vertical: true) + self.metaRow + + if self.skill.disabled { + Text("Disabled in config") + .font(.caption) + .foregroundStyle(.secondary) + } else if !self.requirementsMet, self.shouldShowMissingSummary { + self.missingSummary + } + + if !self.skill.configChecks.isEmpty { + self.configChecksView + } + + if !self.missingEnv.isEmpty { + self.envActionRow + } + } + + Spacer(minLength: 0) + + self.trailingActions + } + .padding(.vertical, 6) + } + + private var sourceLabel: String { + switch self.skill.source { + case "openclaw-bundled": + "Bundled" + case "openclaw-managed": + "Managed" + case "openclaw-workspace": + "Workspace" + case "openclaw-extra": + "Extra" + case "openclaw-plugin": + "Plugin" + default: + self.skill.source + } + } + + private var metaRow: some View { + HStack(spacing: 10) { + SkillTag(text: self.sourceLabel) + if let url = self.homepageUrl { + Link(destination: url) { + Label("Website", systemImage: "link") + .font(.caption2.weight(.semibold)) + } + .buttonStyle(.link) + } + Spacer(minLength: 0) + } + } + + private var homepageUrl: URL? { + guard let raw = self.skill.homepage?.trimmingCharacters(in: .whitespacesAndNewlines) else { + return nil + } + guard !raw.isEmpty else { return nil } + return URL(string: raw) + } + + private var enabledBinding: Binding { + Binding( + get: { !self.skill.disabled }, + set: { self.onToggleEnabled($0) }) + } + + @ViewBuilder + private var missingSummary: some View { + VStack(alignment: .leading, spacing: 4) { + if self.shouldShowMissingBins { + Text("Missing binaries: \(self.missingBins.joined(separator: ", "))") + .font(.caption) + .foregroundStyle(.secondary) + } + if !self.missingEnv.isEmpty { + Text("Missing env: \(self.missingEnv.joined(separator: ", "))") + .font(.caption) + .foregroundStyle(.secondary) + } + if !self.missingConfig.isEmpty { + Text("Requires config: \(self.missingConfig.joined(separator: ", "))") + .font(.caption) + .foregroundStyle(.secondary) + } + } + } + + @ViewBuilder + private var configChecksView: some View { + VStack(alignment: .leading, spacing: 4) { + ForEach(self.skill.configChecks) { check in + HStack(spacing: 6) { + Image(systemName: check.satisfied ? "checkmark.circle" : "xmark.circle") + .foregroundStyle(check.satisfied ? .green : .secondary) + Text(check.path) + .font(.caption) + Text(self.formatConfigValue(check.value)) + .font(.caption) + .foregroundStyle(.secondary) + } + } + } + } + + private var envActionRow: some View { + HStack(spacing: 8) { + ForEach(self.missingEnv, id: \.self) { envKey in + let isPrimary = envKey == self.skill.primaryEnv + Button(isPrimary ? "Set API Key" : "Set \(envKey)") { + self.onSetEnv(envKey, isPrimary) + } + .buttonStyle(.bordered) + .disabled(self.isBusy) + } + Spacer(minLength: 0) + } + } + + @ViewBuilder + private var trailingActions: some View { + VStack(alignment: .trailing, spacing: 8) { + if !self.installOptions.isEmpty { + ForEach(self.installOptions, id: \.id) { (option: SkillInstallOption) in + HStack(spacing: 6) { + if self.showGatewayInstall { + Button("Install on Gateway") { self.onInstall(option, .gateway) } + .buttonStyle(.borderedProminent) + .disabled(self.isBusy) + } + if self.showGatewayInstall { + Button("Install on This Mac") { self.onInstall(option, .local) } + .buttonStyle(.bordered) + .disabled(self.isBusy) + .help( + self.localInstallNeedsSwitch + ? "Switches to Local mode to install on this Mac." + : "") + } else { + Button("Install on This Mac") { self.onInstall(option, .local) } + .buttonStyle(.borderedProminent) + .disabled(self.isBusy) + .help( + self.localInstallNeedsSwitch + ? "Switches to Local mode to install on this Mac." + : "") + } + } + } + } else { + Toggle("", isOn: self.enabledBinding) + .toggleStyle(.switch) + .labelsHidden() + .disabled(self.isBusy || !self.requirementsMet) + } + + if self.isBusy { + ProgressView() + .controlSize(.small) + } + } + } + + private var installOptions: [SkillInstallOption] { + guard !self.missingBins.isEmpty else { return [] } + let missing = Set(self.missingBins) + return self.skill.install.filter { option in + if option.bins.isEmpty { return true } + return !missing.isDisjoint(with: option.bins) + } + } + + private var requirementsMet: Bool { + self.missingBins.isEmpty && self.missingEnv.isEmpty && self.missingConfig.isEmpty + } + + private var shouldShowMissingBins: Bool { + !self.missingBins.isEmpty && self.installOptions.isEmpty + } + + private var shouldShowMissingSummary: Bool { + self.shouldShowMissingBins || + !self.missingEnv.isEmpty || + !self.missingConfig.isEmpty + } + + private var showGatewayInstall: Bool { + self.connectionMode == .remote + } + + private var localInstallNeedsSwitch: Bool { + self.connectionMode != .local + } + + private func formatConfigValue(_ value: AnyCodable?) -> String { + guard let value else { return "" } + switch value.value { + case let bool as Bool: + return bool ? "true" : "false" + case let int as Int: + return String(int) + case let double as Double: + return String(double) + case let string as String: + return string + default: + return "" + } + } +} + +private struct SkillTag: View { + let text: String + + var body: some View { + Text(self.text) + .font(.caption2.weight(.semibold)) + .foregroundStyle(.secondary) + .padding(.horizontal, 8) + .padding(.vertical, 2) + .background(Color.secondary.opacity(0.12)) + .clipShape(Capsule()) + } +} + +private struct EnvEditorState: Identifiable { + let skillKey: String + let skillName: String + let envKey: String + let isPrimary: Bool + + var id: String { "\(self.skillKey)::\(self.envKey)" } +} + +private struct EnvEditorView: View { + let editor: EnvEditorState + let onSave: (String) -> Void + @Environment(\.dismiss) private var dismiss + @State private var value: String = "" + + var body: some View { + VStack(alignment: .leading, spacing: 12) { + Text(self.title) + .font(.headline) + Text(self.subtitle) + .font(.subheadline) + .foregroundStyle(.secondary) + SecureField(self.editor.envKey, text: self.$value) + .textFieldStyle(.roundedBorder) + HStack { + Button("Cancel") { self.dismiss() } + Spacer() + Button("Save") { + self.onSave(self.value) + self.dismiss() + } + .buttonStyle(.borderedProminent) + .disabled(self.value.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty) + } + } + .padding(20) + .frame(width: 420) + } + + private var title: String { + self.editor.isPrimary ? "Set API Key" : "Set Environment Variable" + } + + private var subtitle: String { + "Skill: \(self.editor.skillName)" + } +} + +@MainActor +@Observable +final class SkillsSettingsModel { + var skills: [SkillStatus] = [] + var isLoading = false + var error: String? + var statusMessage: String? + private var busySkills: Set = [] + + func isBusy(skill: SkillStatus) -> Bool { + self.busySkills.contains(skill.skillKey) + } + + func refresh() async { + guard !self.isLoading else { return } + self.isLoading = true + self.error = nil + do { + let report = try await GatewayConnection.shared.skillsStatus() + self.skills = report.skills.sorted { $0.name < $1.name } + } catch { + self.error = error.localizedDescription + } + self.isLoading = false + } + + fileprivate func install(skill: SkillStatus, option: SkillInstallOption, target: InstallTarget) async { + await self.withBusy(skill.skillKey) { + do { + if target == .local, AppStateStore.shared.connectionMode != .local { + AppStateStore.shared.connectionMode = .local + self.statusMessage = "Switched to Local mode to install on this Mac" + } + let result = try await GatewayConnection.shared.skillsInstall( + name: skill.name, + installId: option.id, + timeoutMs: 300_000) + self.statusMessage = result.message + } catch { + self.statusMessage = error.localizedDescription + } + await self.refresh() + } + } + + func setEnabled(skillKey: String, enabled: Bool) async { + await self.withBusy(skillKey) { + do { + _ = try await GatewayConnection.shared.skillsUpdate( + skillKey: skillKey, + enabled: enabled) + self.statusMessage = enabled ? "Skill enabled" : "Skill disabled" + } catch { + self.statusMessage = error.localizedDescription + } + await self.refresh() + } + } + + func updateEnv(skillKey: String, envKey: String, value: String, isPrimary: Bool) async { + await self.withBusy(skillKey) { + do { + if isPrimary { + _ = try await GatewayConnection.shared.skillsUpdate( + skillKey: skillKey, + apiKey: value) + self.statusMessage = "Saved API key" + } else { + _ = try await GatewayConnection.shared.skillsUpdate( + skillKey: skillKey, + env: [envKey: value]) + self.statusMessage = "Saved \(envKey)" + } + } catch { + self.statusMessage = error.localizedDescription + } + await self.refresh() + } + } + + private func withBusy(_ id: String, _ work: @escaping () async -> Void) async { + self.busySkills.insert(id) + defer { self.busySkills.remove(id) } + await work() + } +} + +#if DEBUG +struct SkillsSettings_Previews: PreviewProvider { + static var previews: some View { + SkillsSettings(state: .preview) + .frame(width: SettingsTab.windowWidth, height: SettingsTab.windowHeight) + } +} + +extension SkillsSettings { + static func exerciseForTesting() { + let skill = SkillStatus( + name: "Test Skill", + description: "Test description", + source: "openclaw-bundled", + filePath: "/tmp/skills/test", + baseDir: "/tmp/skills", + skillKey: "test", + primaryEnv: "API_KEY", + emoji: "🧪", + homepage: "https://example.com", + always: false, + disabled: false, + eligible: false, + requirements: SkillRequirements(bins: ["python3"], env: ["API_KEY"], config: ["skills.test"]), + missing: SkillMissing(bins: ["python3"], env: ["API_KEY"], config: ["skills.test"]), + configChecks: [ + SkillStatusConfigCheck(path: "skills.test", value: AnyCodable(false), satisfied: false), + ], + install: [ + SkillInstallOption(id: "brew", kind: "brew", label: "brew install python", bins: ["python3"]), + ]) + + let row = SkillRow( + skill: skill, + isBusy: false, + connectionMode: .remote, + onToggleEnabled: { _ in }, + onInstall: { _, _ in }, + onSetEnv: { _, _ in }) + _ = row.body + + _ = SkillTag(text: "Bundled").body + + let editor = EnvEditorView( + editor: EnvEditorState( + skillKey: "test", + skillName: "Test Skill", + envKey: "API_KEY", + isPrimary: true), + onSave: { _ in }) + _ = editor.body + } + + mutating func setFilterForTesting(_ rawValue: String) { + guard let filter = SkillsFilter(rawValue: rawValue) else { return } + self.filter = filter + } +} +#endif diff --git a/apps/macos/Sources/OpenClaw/SoundEffects.swift b/apps/macos/Sources/OpenClaw/SoundEffects.swift new file mode 100644 index 0000000000000000000000000000000000000000..b321238295df9b509898c1ad25cda1ff5121910e --- /dev/null +++ b/apps/macos/Sources/OpenClaw/SoundEffects.swift @@ -0,0 +1,107 @@ +import AppKit +import Foundation + +enum SoundEffectCatalog { + /// All discoverable system sound names, with "Glass" pinned first. + static var systemOptions: [String] { + var names = Set(Self.discoveredSoundMap.keys).union(Self.fallbackNames) + names.remove("Glass") + let sorted = names.sorted { $0.localizedCaseInsensitiveCompare($1) == .orderedAscending } + return ["Glass"] + sorted + } + + static func displayName(for raw: String) -> String { raw } + + static func url(for name: String) -> URL? { + self.discoveredSoundMap[name] + } + + // MARK: - Internals + + private static let allowedExtensions: Set = [ + "aif", "aiff", "caf", "wav", "m4a", "mp3", + ] + + private static let fallbackNames: [String] = [ + "Glass", // default + "Ping", + "Pop", + "Frog", + "Submarine", + "Funk", + "Tink", + "Basso", + "Blow", + "Bottle", + "Hero", + "Morse", + "Purr", + "Sosumi", + "Mail Sent", + "New Mail", + "Mail Scheduled", + "Mail Fetch Error", + ] + + private static let searchRoots: [URL] = [ + FileManager().homeDirectoryForCurrentUser.appendingPathComponent("Library/Sounds"), + URL(fileURLWithPath: "/Library/Sounds"), + URL(fileURLWithPath: "/System/Applications/Mail.app/Contents/Resources"), // Mail “swoosh” + URL(fileURLWithPath: "/System/Library/Sounds"), + ] + + private static let discoveredSoundMap: [String: URL] = { + var map: [String: URL] = [:] + for root in Self.searchRoots { + guard let contents = try? FileManager().contentsOfDirectory( + at: root, + includingPropertiesForKeys: nil, + options: [.skipsHiddenFiles]) + else { continue } + + for url in contents where Self.allowedExtensions.contains(url.pathExtension.lowercased()) { + let name = url.deletingPathExtension().lastPathComponent + // Preserve the first match in priority order. + if map[name] == nil { + map[name] = url + } + } + } + return map + }() +} + +@MainActor +enum SoundEffectPlayer { + private static var lastSound: NSSound? + + static func sound(named name: String) -> NSSound? { + if let named = NSSound(named: NSSound.Name(name)) { + return named + } + if let url = SoundEffectCatalog.url(for: name) { + return NSSound(contentsOf: url, byReference: false) + } + return nil + } + + static func sound(from bookmark: Data) -> NSSound? { + var stale = false + guard let url = try? URL( + resolvingBookmarkData: bookmark, + options: [.withoutUI, .withSecurityScope], + bookmarkDataIsStale: &stale) + else { return nil } + + let scoped = url.startAccessingSecurityScopedResource() + defer { if scoped { url.stopAccessingSecurityScopedResource() } } + return NSSound(contentsOf: url, byReference: false) + } + + static func play(_ sound: NSSound?) { + guard let sound else { return } + self.lastSound = sound + sound.stop() + sound.play() + } +} diff --git a/apps/macos/Sources/OpenClaw/StatusPill.swift b/apps/macos/Sources/OpenClaw/StatusPill.swift new file mode 100644 index 0000000000000000000000000000000000000000..846ddd419ad26e70c3ef4f1e819ce912a39688b1 --- /dev/null +++ b/apps/macos/Sources/OpenClaw/StatusPill.swift @@ -0,0 +1,16 @@ +import SwiftUI + +struct StatusPill: View { + let text: String + let tint: Color + + var body: some View { + Text(self.text) + .font(.caption2.weight(.semibold)) + .padding(.horizontal, 7) + .padding(.vertical, 3) + .foregroundStyle(self.tint == .secondary ? .secondary : self.tint) + .background((self.tint == .secondary ? Color.secondary : self.tint).opacity(0.12)) + .clipShape(Capsule()) + } +} diff --git a/apps/macos/Sources/OpenClaw/String+NonEmpty.swift b/apps/macos/Sources/OpenClaw/String+NonEmpty.swift new file mode 100644 index 0000000000000000000000000000000000000000..402e4c2db5f55aa057b2ece630c7031f93a2d410 --- /dev/null +++ b/apps/macos/Sources/OpenClaw/String+NonEmpty.swift @@ -0,0 +1,8 @@ +import Foundation + +extension String { + var nonEmpty: String? { + let trimmed = self.trimmingCharacters(in: .whitespacesAndNewlines) + return trimmed.isEmpty ? nil : trimmed + } +} diff --git a/apps/macos/Sources/OpenClaw/SystemRunSettingsView.swift b/apps/macos/Sources/OpenClaw/SystemRunSettingsView.swift new file mode 100644 index 0000000000000000000000000000000000000000..eef826c3f0c7172f746c77a1f6ab3c363f831bf5 --- /dev/null +++ b/apps/macos/Sources/OpenClaw/SystemRunSettingsView.swift @@ -0,0 +1,412 @@ +import Foundation +import Observation +import SwiftUI + +struct SystemRunSettingsView: View { + @State private var model = ExecApprovalsSettingsModel() + @State private var tab: ExecApprovalsSettingsTab = .policy + @State private var newPattern: String = "" + + var body: some View { + VStack(alignment: .leading, spacing: 8) { + HStack(alignment: .center, spacing: 12) { + Text("Exec approvals") + .font(.body) + Spacer(minLength: 0) + Picker("Agent", selection: Binding( + get: { self.model.selectedAgentId }, + set: { self.model.selectAgent($0) })) + { + ForEach(self.model.agentPickerIds, id: \.self) { id in + Text(self.model.label(for: id)).tag(id) + } + } + .pickerStyle(.menu) + .frame(width: 180, alignment: .trailing) + } + + Picker("", selection: self.$tab) { + ForEach(ExecApprovalsSettingsTab.allCases) { tab in + Text(tab.title).tag(tab) + } + } + .pickerStyle(.segmented) + .frame(width: 320) + + if self.tab == .policy { + self.policyView + } else { + self.allowlistView + } + } + .task { await self.model.refresh() } + .onChange(of: self.tab) { _, _ in + Task { await self.model.refreshSkillBins() } + } + } + + private var policyView: some View { + VStack(alignment: .leading, spacing: 8) { + Picker("", selection: Binding( + get: { self.model.security }, + set: { self.model.setSecurity($0) })) + { + ForEach(ExecSecurity.allCases) { security in + Text(security.title).tag(security) + } + } + .labelsHidden() + .pickerStyle(.menu) + + Picker("", selection: Binding( + get: { self.model.ask }, + set: { self.model.setAsk($0) })) + { + ForEach(ExecAsk.allCases) { ask in + Text(ask.title).tag(ask) + } + } + .labelsHidden() + .pickerStyle(.menu) + + Picker("", selection: Binding( + get: { self.model.askFallback }, + set: { self.model.setAskFallback($0) })) + { + ForEach(ExecSecurity.allCases) { mode in + Text("Fallback: \(mode.title)").tag(mode) + } + } + .labelsHidden() + .pickerStyle(.menu) + + Text(self.scopeMessage) + .font(.footnote) + .foregroundStyle(.tertiary) + .fixedSize(horizontal: false, vertical: true) + } + } + + private var allowlistView: some View { + VStack(alignment: .leading, spacing: 10) { + Toggle("Auto-allow skill CLIs", isOn: Binding( + get: { self.model.autoAllowSkills }, + set: { self.model.setAutoAllowSkills($0) })) + + if self.model.autoAllowSkills, !self.model.skillBins.isEmpty { + Text("Skill CLIs: \(self.model.skillBins.joined(separator: ", "))") + .font(.footnote) + .foregroundStyle(.secondary) + } + + if self.model.isDefaultsScope { + Text("Allowlists are per-agent. Select an agent to edit its allowlist.") + .font(.footnote) + .foregroundStyle(.secondary) + } else { + HStack(spacing: 8) { + TextField("Add allowlist pattern (case-insensitive globs)", text: self.$newPattern) + .textFieldStyle(.roundedBorder) + Button("Add") { + let pattern = self.newPattern.trimmingCharacters(in: .whitespacesAndNewlines) + guard !pattern.isEmpty else { return } + self.model.addEntry(pattern) + self.newPattern = "" + } + .buttonStyle(.bordered) + .disabled(self.newPattern.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty) + } + + if self.model.entries.isEmpty { + Text("No allowlisted commands yet.") + .font(.footnote) + .foregroundStyle(.secondary) + } else { + VStack(alignment: .leading, spacing: 8) { + ForEach(self.model.entries, id: \.id) { entry in + ExecAllowlistRow( + entry: Binding( + get: { self.model.entry(for: entry.id) ?? entry }, + set: { self.model.updateEntry($0, id: entry.id) }), + onRemove: { self.model.removeEntry(id: entry.id) }) + } + } + } + } + } + } + + private var scopeMessage: String { + if self.model.isDefaultsScope { + return "Defaults apply when an agent has no overrides. " + + "Ask controls prompt behavior; fallback is used when no companion UI is reachable." + } + return "Security controls whether system.run can execute on this Mac when paired as a node. " + + "Ask controls prompt behavior; fallback is used when no companion UI is reachable." + } +} + +private enum ExecApprovalsSettingsTab: String, CaseIterable, Identifiable { + case policy + case allowlist + + var id: String { self.rawValue } + + var title: String { + switch self { + case .policy: "Access" + case .allowlist: "Allowlist" + } + } +} + +struct ExecAllowlistRow: View { + @Binding var entry: ExecAllowlistEntry + let onRemove: () -> Void + @State private var draftPattern: String = "" + + private static let relativeFormatter: RelativeDateTimeFormatter = { + let formatter = RelativeDateTimeFormatter() + formatter.unitsStyle = .short + return formatter + }() + + var body: some View { + VStack(alignment: .leading, spacing: 4) { + HStack(spacing: 8) { + TextField("Pattern", text: self.patternBinding) + .textFieldStyle(.roundedBorder) + + Button(role: .destructive) { + self.onRemove() + } label: { + Image(systemName: "trash") + } + .buttonStyle(.borderless) + } + + if let lastUsedAt = self.entry.lastUsedAt { + let date = Date(timeIntervalSince1970: lastUsedAt / 1000.0) + Text("Last used \(Self.relativeFormatter.localizedString(for: date, relativeTo: Date()))") + .font(.caption) + .foregroundStyle(.secondary) + } + + if let lastUsedCommand = self.entry.lastUsedCommand, !lastUsedCommand.isEmpty { + Text("Last command: \(lastUsedCommand)") + .font(.caption) + .foregroundStyle(.secondary) + } + + if let lastResolvedPath = self.entry.lastResolvedPath, !lastResolvedPath.isEmpty { + Text("Resolved path: \(lastResolvedPath)") + .font(.caption) + .foregroundStyle(.secondary) + } + } + .onAppear { + self.draftPattern = self.entry.pattern + } + } + + private var patternBinding: Binding { + Binding( + get: { self.draftPattern.isEmpty ? self.entry.pattern : self.draftPattern }, + set: { newValue in + self.draftPattern = newValue + self.entry.pattern = newValue + }) + } +} + +@MainActor +@Observable +final class ExecApprovalsSettingsModel { + private static let defaultsScopeId = "__defaults__" + var agentIds: [String] = [] + var selectedAgentId: String = "main" + var defaultAgentId: String = "main" + var security: ExecSecurity = .deny + var ask: ExecAsk = .onMiss + var askFallback: ExecSecurity = .deny + var autoAllowSkills = false + var entries: [ExecAllowlistEntry] = [] + var skillBins: [String] = [] + + var agentPickerIds: [String] { + [Self.defaultsScopeId] + self.agentIds + } + + var isDefaultsScope: Bool { + self.selectedAgentId == Self.defaultsScopeId + } + + func label(for id: String) -> String { + if id == Self.defaultsScopeId { return "Defaults" } + return id + } + + func refresh() async { + await self.refreshAgents() + self.loadSettings(for: self.selectedAgentId) + await self.refreshSkillBins() + } + + func refreshAgents() async { + let root = await ConfigStore.load() + let agents = root["agents"] as? [String: Any] + let list = agents?["list"] as? [[String: Any]] ?? [] + var ids: [String] = [] + var seen = Set() + var defaultId: String? + for entry in list { + guard let raw = entry["id"] as? String else { continue } + let trimmed = raw.trimmingCharacters(in: .whitespacesAndNewlines) + guard !trimmed.isEmpty else { continue } + if !seen.insert(trimmed).inserted { continue } + ids.append(trimmed) + if (entry["default"] as? Bool) == true, defaultId == nil { + defaultId = trimmed + } + } + if ids.isEmpty { + ids = ["main"] + defaultId = "main" + } else if defaultId == nil { + defaultId = ids.first + } + self.agentIds = ids + self.defaultAgentId = defaultId ?? "main" + if self.selectedAgentId == Self.defaultsScopeId { + return + } + if !self.agentIds.contains(self.selectedAgentId) { + self.selectedAgentId = self.defaultAgentId + } + } + + func selectAgent(_ id: String) { + self.selectedAgentId = id + self.loadSettings(for: id) + Task { await self.refreshSkillBins() } + } + + func loadSettings(for agentId: String) { + if agentId == Self.defaultsScopeId { + let defaults = ExecApprovalsStore.resolveDefaults() + self.security = defaults.security + self.ask = defaults.ask + self.askFallback = defaults.askFallback + self.autoAllowSkills = defaults.autoAllowSkills + self.entries = [] + return + } + let resolved = ExecApprovalsStore.resolve(agentId: agentId) + self.security = resolved.agent.security + self.ask = resolved.agent.ask + self.askFallback = resolved.agent.askFallback + self.autoAllowSkills = resolved.agent.autoAllowSkills + self.entries = resolved.allowlist + .sorted { $0.pattern.localizedCaseInsensitiveCompare($1.pattern) == .orderedAscending } + } + + func setSecurity(_ security: ExecSecurity) { + self.security = security + if self.isDefaultsScope { + ExecApprovalsStore.updateDefaults { defaults in + defaults.security = security + } + } else { + ExecApprovalsStore.updateAgentSettings(agentId: self.selectedAgentId) { entry in + entry.security = security + } + } + self.syncQuickMode() + } + + func setAsk(_ ask: ExecAsk) { + self.ask = ask + if self.isDefaultsScope { + ExecApprovalsStore.updateDefaults { defaults in + defaults.ask = ask + } + } else { + ExecApprovalsStore.updateAgentSettings(agentId: self.selectedAgentId) { entry in + entry.ask = ask + } + } + self.syncQuickMode() + } + + func setAskFallback(_ mode: ExecSecurity) { + self.askFallback = mode + if self.isDefaultsScope { + ExecApprovalsStore.updateDefaults { defaults in + defaults.askFallback = mode + } + } else { + ExecApprovalsStore.updateAgentSettings(agentId: self.selectedAgentId) { entry in + entry.askFallback = mode + } + } + } + + func setAutoAllowSkills(_ enabled: Bool) { + self.autoAllowSkills = enabled + if self.isDefaultsScope { + ExecApprovalsStore.updateDefaults { defaults in + defaults.autoAllowSkills = enabled + } + } else { + ExecApprovalsStore.updateAgentSettings(agentId: self.selectedAgentId) { entry in + entry.autoAllowSkills = enabled + } + } + Task { await self.refreshSkillBins(force: enabled) } + } + + func addEntry(_ pattern: String) { + guard !self.isDefaultsScope else { return } + let trimmed = pattern.trimmingCharacters(in: .whitespacesAndNewlines) + guard !trimmed.isEmpty else { return } + self.entries.append(ExecAllowlistEntry(pattern: trimmed, lastUsedAt: nil)) + ExecApprovalsStore.updateAllowlist(agentId: self.selectedAgentId, allowlist: self.entries) + } + + func updateEntry(_ entry: ExecAllowlistEntry, id: UUID) { + guard !self.isDefaultsScope else { return } + guard let index = self.entries.firstIndex(where: { $0.id == id }) else { return } + self.entries[index] = entry + ExecApprovalsStore.updateAllowlist(agentId: self.selectedAgentId, allowlist: self.entries) + } + + func removeEntry(id: UUID) { + guard !self.isDefaultsScope else { return } + guard let index = self.entries.firstIndex(where: { $0.id == id }) else { return } + self.entries.remove(at: index) + ExecApprovalsStore.updateAllowlist(agentId: self.selectedAgentId, allowlist: self.entries) + } + + func entry(for id: UUID) -> ExecAllowlistEntry? { + self.entries.first(where: { $0.id == id }) + } + + func refreshSkillBins(force: Bool = false) async { + guard self.autoAllowSkills else { + self.skillBins = [] + return + } + let bins = await SkillBinsCache.shared.currentBins(force: force) + self.skillBins = bins.sorted() + } + + private func syncQuickMode() { + if self.isDefaultsScope { + AppStateStore.shared.execApprovalMode = ExecApprovalQuickMode.from(security: self.security, ask: self.ask) + return + } + if self.selectedAgentId == self.defaultAgentId || self.agentIds.count <= 1 { + AppStateStore.shared.execApprovalMode = ExecApprovalQuickMode.from(security: self.security, ask: self.ask) + } + } +} diff --git a/apps/macos/Sources/OpenClaw/TailscaleIntegrationSection.swift b/apps/macos/Sources/OpenClaw/TailscaleIntegrationSection.swift new file mode 100644 index 0000000000000000000000000000000000000000..c1a3a3489a69dbea5b174a03ddc5482a939a804b --- /dev/null +++ b/apps/macos/Sources/OpenClaw/TailscaleIntegrationSection.swift @@ -0,0 +1,399 @@ +import SwiftUI + +private enum GatewayTailscaleMode: String, CaseIterable, Identifiable { + case off + case serve + case funnel + + var id: String { self.rawValue } + + var label: String { + switch self { + case .off: "Off" + case .serve: "Tailnet (Serve)" + case .funnel: "Public (Funnel)" + } + } + + var description: String { + switch self { + case .off: + "No automatic Tailscale configuration." + case .serve: + "Tailnet-only HTTPS via Tailscale Serve." + case .funnel: + "Public HTTPS via Tailscale Funnel (requires auth)." + } + } +} + +struct TailscaleIntegrationSection: View { + let connectionMode: AppState.ConnectionMode + let isPaused: Bool + + @Environment(TailscaleService.self) private var tailscaleService + #if DEBUG + private var testingService: TailscaleService? + #endif + + @State private var hasLoaded = false + @State private var tailscaleMode: GatewayTailscaleMode = .serve + @State private var requireCredentialsForServe = false + @State private var password: String = "" + @State private var statusMessage: String? + @State private var validationMessage: String? + @State private var statusTimer: Timer? + + init(connectionMode: AppState.ConnectionMode, isPaused: Bool) { + self.connectionMode = connectionMode + self.isPaused = isPaused + #if DEBUG + self.testingService = nil + #endif + } + + private var effectiveService: TailscaleService { + #if DEBUG + return self.testingService ?? self.tailscaleService + #else + return self.tailscaleService + #endif + } + + var body: some View { + VStack(alignment: .leading, spacing: 10) { + Text("Tailscale (dashboard access)") + .font(.callout.weight(.semibold)) + + self.statusRow + + if !self.effectiveService.isInstalled { + self.installButtons + } else { + self.modePicker + if self.tailscaleMode != .off { + self.accessURLRow + } + if self.tailscaleMode == .serve { + self.serveAuthSection + } + if self.tailscaleMode == .funnel { + self.funnelAuthSection + } + } + + if self.connectionMode != .local { + Text("Local mode required. Update settings on the gateway host.") + .font(.caption) + .foregroundStyle(.secondary) + } + + if let validationMessage { + Text(validationMessage) + .font(.caption) + .foregroundStyle(.orange) + } else if let statusMessage { + Text(statusMessage) + .font(.caption) + .foregroundStyle(.secondary) + } + } + .padding(12) + .background(Color.gray.opacity(0.08)) + .cornerRadius(10) + .disabled(self.connectionMode != .local) + .task { + guard !self.hasLoaded else { return } + await self.loadConfig() + self.hasLoaded = true + await self.effectiveService.checkTailscaleStatus() + self.startStatusTimer() + } + .onDisappear { + self.stopStatusTimer() + } + .onChange(of: self.tailscaleMode) { _, _ in + Task { await self.applySettings() } + } + .onChange(of: self.requireCredentialsForServe) { _, _ in + Task { await self.applySettings() } + } + } + + private var statusRow: some View { + HStack(spacing: 8) { + Circle() + .fill(self.statusColor) + .frame(width: 10, height: 10) + Text(self.statusText) + .font(.callout) + Spacer() + Button("Refresh") { + Task { await self.effectiveService.checkTailscaleStatus() } + } + .buttonStyle(.bordered) + .controlSize(.small) + } + } + + private var statusColor: Color { + if !self.effectiveService.isInstalled { return .yellow } + if self.effectiveService.isRunning { return .green } + return .orange + } + + private var statusText: String { + if !self.effectiveService.isInstalled { return "Tailscale is not installed" } + if self.effectiveService.isRunning { return "Tailscale is installed and running" } + return "Tailscale is installed but not running" + } + + private var installButtons: some View { + HStack(spacing: 12) { + Button("App Store") { self.effectiveService.openAppStore() } + .buttonStyle(.link) + Button("Direct Download") { self.effectiveService.openDownloadPage() } + .buttonStyle(.link) + Button("Setup Guide") { self.effectiveService.openSetupGuide() } + .buttonStyle(.link) + } + .controlSize(.small) + } + + private var modePicker: some View { + VStack(alignment: .leading, spacing: 6) { + Text("Exposure mode") + .font(.callout.weight(.semibold)) + Picker("Exposure", selection: self.$tailscaleMode) { + ForEach(GatewayTailscaleMode.allCases) { mode in + Text(mode.label).tag(mode) + } + } + .pickerStyle(.segmented) + Text(self.tailscaleMode.description) + .font(.caption) + .foregroundStyle(.secondary) + } + } + + @ViewBuilder + private var accessURLRow: some View { + if let host = self.effectiveService.tailscaleHostname { + let url = "https://\(host)/ui/" + HStack(spacing: 8) { + Text("Dashboard URL:") + .font(.caption) + .foregroundStyle(.secondary) + if let link = URL(string: url) { + Link(url, destination: link) + .font(.system(.caption, design: .monospaced)) + } else { + Text(url) + .font(.system(.caption, design: .monospaced)) + } + } + } else if !self.effectiveService.isRunning { + Text("Start Tailscale to get your tailnet hostname.") + .font(.caption) + .foregroundStyle(.secondary) + } + + if self.effectiveService.isInstalled, !self.effectiveService.isRunning { + Button("Start Tailscale") { self.effectiveService.openTailscaleApp() } + .buttonStyle(.borderedProminent) + .controlSize(.small) + } + } + + private var serveAuthSection: some View { + VStack(alignment: .leading, spacing: 8) { + Toggle("Require credentials", isOn: self.$requireCredentialsForServe) + .toggleStyle(.checkbox) + if self.requireCredentialsForServe { + self.authFields + } else { + Text("Serve uses Tailscale identity headers; no password required.") + .font(.caption) + .foregroundStyle(.secondary) + } + } + } + + private var funnelAuthSection: some View { + VStack(alignment: .leading, spacing: 8) { + Text("Funnel requires authentication.") + .font(.caption) + .foregroundStyle(.secondary) + self.authFields + } + } + + @ViewBuilder + private var authFields: some View { + SecureField("Password", text: self.$password) + .textFieldStyle(.roundedBorder) + .frame(maxWidth: 240) + .onSubmit { Task { await self.applySettings() } } + Text("Stored in ~/.openclaw/openclaw.json. Prefer OPENCLAW_GATEWAY_PASSWORD for production.") + .font(.caption) + .foregroundStyle(.secondary) + Button("Update password") { Task { await self.applySettings() } } + .buttonStyle(.bordered) + .controlSize(.small) + } + + private func loadConfig() async { + let root = await ConfigStore.load() + let gateway = root["gateway"] as? [String: Any] ?? [:] + let tailscale = gateway["tailscale"] as? [String: Any] ?? [:] + let modeRaw = (tailscale["mode"] as? String) ?? "serve" + self.tailscaleMode = GatewayTailscaleMode(rawValue: modeRaw) ?? .off + + let auth = gateway["auth"] as? [String: Any] ?? [:] + let authModeRaw = auth["mode"] as? String + let allowTailscale = auth["allowTailscale"] as? Bool + + self.password = auth["password"] as? String ?? "" + + if self.tailscaleMode == .serve { + let usesExplicitAuth = authModeRaw == "password" + if let allowTailscale, allowTailscale == false { + self.requireCredentialsForServe = true + } else { + self.requireCredentialsForServe = usesExplicitAuth + } + } else { + self.requireCredentialsForServe = false + } + } + + private func applySettings() async { + guard self.hasLoaded else { return } + self.validationMessage = nil + self.statusMessage = nil + + let trimmedPassword = self.password.trimmingCharacters(in: .whitespacesAndNewlines) + let requiresPassword = self.tailscaleMode == .funnel + || (self.tailscaleMode == .serve && self.requireCredentialsForServe) + if requiresPassword, trimmedPassword.isEmpty { + self.validationMessage = "Password required for this mode." + return + } + + let (success, errorMessage) = await TailscaleIntegrationSection.buildAndSaveTailscaleConfig( + tailscaleMode: self.tailscaleMode, + requireCredentialsForServe: self.requireCredentialsForServe, + password: trimmedPassword, + connectionMode: self.connectionMode, + isPaused: self.isPaused) + + if !success, let errorMessage { + self.statusMessage = errorMessage + return + } + + if self.connectionMode == .local, !self.isPaused { + self.statusMessage = "Saved to ~/.openclaw/openclaw.json. Restarting gateway…" + } else { + self.statusMessage = "Saved to ~/.openclaw/openclaw.json. Restart the gateway to apply." + } + self.restartGatewayIfNeeded() + } + + @MainActor + private static func buildAndSaveTailscaleConfig( + tailscaleMode: GatewayTailscaleMode, + requireCredentialsForServe: Bool, + password: String, + connectionMode: AppState.ConnectionMode, + isPaused: Bool) async -> (Bool, String?) + { + var root = await ConfigStore.load() + var gateway = root["gateway"] as? [String: Any] ?? [:] + var tailscale = gateway["tailscale"] as? [String: Any] ?? [:] + tailscale["mode"] = tailscaleMode.rawValue + gateway["tailscale"] = tailscale + + if tailscaleMode != .off { + gateway["bind"] = "loopback" + } + + if tailscaleMode == .off { + gateway.removeValue(forKey: "auth") + } else { + var auth = gateway["auth"] as? [String: Any] ?? [:] + if tailscaleMode == .serve, !requireCredentialsForServe { + auth["allowTailscale"] = true + auth.removeValue(forKey: "mode") + auth.removeValue(forKey: "password") + } else { + auth["allowTailscale"] = false + auth["mode"] = "password" + auth["password"] = password + } + + if auth.isEmpty { + gateway.removeValue(forKey: "auth") + } else { + gateway["auth"] = auth + } + } + + if gateway.isEmpty { + root.removeValue(forKey: "gateway") + } else { + root["gateway"] = gateway + } + + do { + try await ConfigStore.save(root) + return (true, nil) + } catch { + return (false, error.localizedDescription) + } + } + + private func restartGatewayIfNeeded() { + guard self.connectionMode == .local, !self.isPaused else { return } + Task { await GatewayLaunchAgentManager.kickstart() } + } + + private func startStatusTimer() { + self.stopStatusTimer() + if ProcessInfo.processInfo.isRunningTests { + return + } + self.statusTimer = Timer.scheduledTimer(withTimeInterval: 5, repeats: true) { _ in + Task { await self.effectiveService.checkTailscaleStatus() } + } + } + + private func stopStatusTimer() { + self.statusTimer?.invalidate() + self.statusTimer = nil + } +} + +#if DEBUG +extension TailscaleIntegrationSection { + mutating func setTestingState( + mode: String, + requireCredentials: Bool, + password: String = "secret", + statusMessage: String? = nil, + validationMessage: String? = nil) + { + if let mode = GatewayTailscaleMode(rawValue: mode) { + self.tailscaleMode = mode + } + self.requireCredentialsForServe = requireCredentials + self.password = password + self.statusMessage = statusMessage + self.validationMessage = validationMessage + } + + mutating func setTestingService(_ service: TailscaleService?) { + self.testingService = service + } +} +#endif diff --git a/apps/macos/Sources/OpenClaw/TailscaleService.swift b/apps/macos/Sources/OpenClaw/TailscaleService.swift new file mode 100644 index 0000000000000000000000000000000000000000..b7f716a4270475c82e8494d1f204b6f4a9acf459 --- /dev/null +++ b/apps/macos/Sources/OpenClaw/TailscaleService.swift @@ -0,0 +1,226 @@ +import AppKit +import Foundation +import Observation +import os +#if canImport(Darwin) +import Darwin +#endif + +/// Manages Tailscale integration and status checking. +@Observable +@MainActor +final class TailscaleService { + static let shared = TailscaleService() + + /// Tailscale local API endpoint. + private static let tailscaleAPIEndpoint = "http://100.100.100.100/api/data" + + /// API request timeout in seconds. + private static let apiTimeoutInterval: TimeInterval = 5.0 + + private let logger = Logger(subsystem: "ai.openclaw", category: "tailscale") + + /// Indicates if the Tailscale app is installed on the system. + private(set) var isInstalled = false + + /// Indicates if Tailscale is currently running. + private(set) var isRunning = false + + /// The Tailscale hostname for this device (e.g., "my-mac.tailnet.ts.net"). + private(set) var tailscaleHostname: String? + + /// The Tailscale IPv4 address for this device. + private(set) var tailscaleIP: String? + + /// Error message if status check fails. + private(set) var statusError: String? + + private init() { + Task { await self.checkTailscaleStatus() } + } + + #if DEBUG + init( + isInstalled: Bool, + isRunning: Bool, + tailscaleHostname: String? = nil, + tailscaleIP: String? = nil, + statusError: String? = nil) + { + self.isInstalled = isInstalled + self.isRunning = isRunning + self.tailscaleHostname = tailscaleHostname + self.tailscaleIP = tailscaleIP + self.statusError = statusError + } + #endif + + func checkAppInstallation() -> Bool { + let installed = FileManager().fileExists(atPath: "/Applications/Tailscale.app") + self.logger.info("Tailscale app installed: \(installed)") + return installed + } + + private struct TailscaleAPIResponse: Codable { + let status: String + let deviceName: String + let tailnetName: String + let iPv4: String? + + private enum CodingKeys: String, CodingKey { + case status = "Status" + case deviceName = "DeviceName" + case tailnetName = "TailnetName" + case iPv4 = "IPv4" + } + } + + private func fetchTailscaleStatus() async -> TailscaleAPIResponse? { + guard let url = URL(string: Self.tailscaleAPIEndpoint) else { + self.logger.error("Invalid Tailscale API URL") + return nil + } + + do { + let configuration = URLSessionConfiguration.default + configuration.timeoutIntervalForRequest = Self.apiTimeoutInterval + let session = URLSession(configuration: configuration) + + let (data, response) = try await session.data(from: url) + guard let httpResponse = response as? HTTPURLResponse, + httpResponse.statusCode == 200 + else { + self.logger.warning("Tailscale API returned non-200 status") + return nil + } + + let decoder = JSONDecoder() + return try decoder.decode(TailscaleAPIResponse.self, from: data) + } catch { + self.logger.debug("Failed to fetch Tailscale status: \(String(describing: error))") + return nil + } + } + + func checkTailscaleStatus() async { + let previousIP = self.tailscaleIP + self.isInstalled = self.checkAppInstallation() + if !self.isInstalled { + self.isRunning = false + self.tailscaleHostname = nil + self.tailscaleIP = nil + self.statusError = "Tailscale is not installed" + } else if let apiResponse = await fetchTailscaleStatus() { + self.isRunning = apiResponse.status.lowercased() == "running" + + if self.isRunning { + let deviceName = apiResponse.deviceName + .lowercased() + .replacingOccurrences(of: " ", with: "-") + let tailnetName = apiResponse.tailnetName + .replacingOccurrences(of: ".ts.net", with: "") + .replacingOccurrences(of: ".tailscale.net", with: "") + + self.tailscaleHostname = "\(deviceName).\(tailnetName).ts.net" + self.tailscaleIP = apiResponse.iPv4 + self.statusError = nil + + self.logger.info( + "Tailscale running host=\(self.tailscaleHostname ?? "nil") ip=\(self.tailscaleIP ?? "nil")") + } else { + self.tailscaleHostname = nil + self.tailscaleIP = nil + self.statusError = "Tailscale is not running" + } + } else { + self.isRunning = false + self.tailscaleHostname = nil + self.tailscaleIP = nil + self.statusError = "Please start the Tailscale app" + self.logger.info("Tailscale API not responding; app likely not running") + } + + if self.tailscaleIP == nil, let fallback = Self.detectTailnetIPv4() { + self.tailscaleIP = fallback + if !self.isRunning { + self.isRunning = true + } + self.statusError = nil + self.logger.info("Tailscale interface IP detected (fallback) ip=\(fallback, privacy: .public)") + } + + if previousIP != self.tailscaleIP { + await GatewayEndpointStore.shared.refresh() + } + } + + func openTailscaleApp() { + if let url = URL(string: "file:///Applications/Tailscale.app") { + NSWorkspace.shared.open(url) + } + } + + func openAppStore() { + if let url = URL(string: "https://apps.apple.com/us/app/tailscale/id1475387142") { + NSWorkspace.shared.open(url) + } + } + + func openDownloadPage() { + if let url = URL(string: "https://tailscale.com/download/macos") { + NSWorkspace.shared.open(url) + } + } + + func openSetupGuide() { + if let url = URL(string: "https://tailscale.com/kb/1017/install/") { + NSWorkspace.shared.open(url) + } + } + + private nonisolated static func isTailnetIPv4(_ address: String) -> Bool { + let parts = address.split(separator: ".") + guard parts.count == 4 else { return false } + let octets = parts.compactMap { Int($0) } + guard octets.count == 4 else { return false } + let a = octets[0] + let b = octets[1] + return a == 100 && b >= 64 && b <= 127 + } + + private nonisolated static func detectTailnetIPv4() -> String? { + var addrList: UnsafeMutablePointer? + guard getifaddrs(&addrList) == 0, let first = addrList else { return nil } + defer { freeifaddrs(addrList) } + + for ptr in sequence(first: first, next: { $0.pointee.ifa_next }) { + let flags = Int32(ptr.pointee.ifa_flags) + let isUp = (flags & IFF_UP) != 0 + let isLoopback = (flags & IFF_LOOPBACK) != 0 + let family = ptr.pointee.ifa_addr.pointee.sa_family + if !isUp || isLoopback || family != UInt8(AF_INET) { continue } + + var addr = ptr.pointee.ifa_addr.pointee + var buffer = [CChar](repeating: 0, count: Int(NI_MAXHOST)) + let result = getnameinfo( + &addr, + socklen_t(ptr.pointee.ifa_addr.pointee.sa_len), + &buffer, + socklen_t(buffer.count), + nil, + 0, + NI_NUMERICHOST) + guard result == 0 else { continue } + let len = buffer.prefix { $0 != 0 } + let bytes = len.map { UInt8(bitPattern: $0) } + guard let ip = String(bytes: bytes, encoding: .utf8) else { continue } + if Self.isTailnetIPv4(ip) { return ip } + } + + return nil + } + + nonisolated static func fallbackTailnetIPv4() -> String? { + self.detectTailnetIPv4() + } +} diff --git a/apps/macos/Sources/OpenClaw/TalkAudioPlayer.swift b/apps/macos/Sources/OpenClaw/TalkAudioPlayer.swift new file mode 100644 index 0000000000000000000000000000000000000000..ae9a06451044e8853a4371fa68967849f8efa8b6 --- /dev/null +++ b/apps/macos/Sources/OpenClaw/TalkAudioPlayer.swift @@ -0,0 +1,158 @@ +import AVFoundation +import Foundation +import OSLog + +@MainActor +final class TalkAudioPlayer: NSObject, @preconcurrency AVAudioPlayerDelegate { + static let shared = TalkAudioPlayer() + + private let logger = Logger(subsystem: "ai.openclaw", category: "talk.tts") + private var player: AVAudioPlayer? + private var playback: Playback? + + private final class Playback: @unchecked Sendable { + private let lock = NSLock() + private var finished = false + private var continuation: CheckedContinuation? + private var watchdog: Task? + + func setContinuation(_ continuation: CheckedContinuation) { + self.lock.lock() + defer { self.lock.unlock() } + self.continuation = continuation + } + + func setWatchdog(_ task: Task?) { + self.lock.lock() + let old = self.watchdog + self.watchdog = task + self.lock.unlock() + old?.cancel() + } + + func cancelWatchdog() { + self.setWatchdog(nil) + } + + func finish(_ result: TalkPlaybackResult) { + let continuation: CheckedContinuation? + self.lock.lock() + if self.finished { + continuation = nil + } else { + self.finished = true + continuation = self.continuation + self.continuation = nil + } + self.lock.unlock() + continuation?.resume(returning: result) + } + } + + func play(data: Data) async -> TalkPlaybackResult { + self.stopInternal() + + let playback = Playback() + self.playback = playback + + return await withCheckedContinuation { continuation in + playback.setContinuation(continuation) + do { + let player = try AVAudioPlayer(data: data) + self.player = player + + player.delegate = self + player.prepareToPlay() + + self.armWatchdog(playback: playback) + + let ok = player.play() + if !ok { + self.logger.error("talk audio player refused to play") + self.finish(playback: playback, result: TalkPlaybackResult(finished: false, interruptedAt: nil)) + } + } catch { + self.logger.error("talk audio player failed: \(error.localizedDescription, privacy: .public)") + self.finish(playback: playback, result: TalkPlaybackResult(finished: false, interruptedAt: nil)) + } + } + } + + func stop() -> Double? { + guard let player else { return nil } + let time = player.currentTime + self.stopInternal(interruptedAt: time) + return time + } + + func audioPlayerDidFinishPlaying(_: AVAudioPlayer, successfully flag: Bool) { + self.stopInternal(finished: flag) + } + + private func stopInternal(finished: Bool = false, interruptedAt: Double? = nil) { + guard let playback else { return } + let result = TalkPlaybackResult(finished: finished, interruptedAt: interruptedAt) + self.finish(playback: playback, result: result) + } + + private func finish(playback: Playback, result: TalkPlaybackResult) { + playback.cancelWatchdog() + playback.finish(result) + + guard self.playback === playback else { return } + self.playback = nil + self.player?.stop() + self.player = nil + } + + private func stopInternal() { + if let playback = self.playback { + let interruptedAt = self.player?.currentTime + self.finish( + playback: playback, + result: TalkPlaybackResult(finished: false, interruptedAt: interruptedAt)) + return + } + self.player?.stop() + self.player = nil + } + + private func armWatchdog(playback: Playback) { + playback.setWatchdog(Task { @MainActor [weak self] in + guard let self else { return } + + do { + try await Task.sleep(nanoseconds: 650_000_000) + } catch { + return + } + if Task.isCancelled { return } + + guard self.playback === playback else { return } + if self.player?.isPlaying != true { + self.logger.error("talk audio player did not start playing") + self.finish(playback: playback, result: TalkPlaybackResult(finished: false, interruptedAt: nil)) + return + } + + let duration = self.player?.duration ?? 0 + let timeoutSeconds = min(max(2.0, duration + 2.0), 5 * 60.0) + do { + try await Task.sleep(nanoseconds: UInt64(timeoutSeconds * 1_000_000_000)) + } catch { + return + } + if Task.isCancelled { return } + + guard self.playback === playback else { return } + guard self.player?.isPlaying == true else { return } + self.logger.error("talk audio player watchdog fired") + self.finish(playback: playback, result: TalkPlaybackResult(finished: false, interruptedAt: nil)) + }) + } +} + +struct TalkPlaybackResult: Sendable { + let finished: Bool + let interruptedAt: Double? +} diff --git a/apps/macos/Sources/OpenClaw/TalkModeController.swift b/apps/macos/Sources/OpenClaw/TalkModeController.swift new file mode 100644 index 0000000000000000000000000000000000000000..8454e503b4fab8788b425084ef73a829c6577623 --- /dev/null +++ b/apps/macos/Sources/OpenClaw/TalkModeController.swift @@ -0,0 +1,69 @@ +import Observation + +@MainActor +@Observable +final class TalkModeController { + static let shared = TalkModeController() + + private let logger = Logger(subsystem: "ai.openclaw", category: "talk.controller") + + private(set) var phase: TalkModePhase = .idle + private(set) var isPaused: Bool = false + + func setEnabled(_ enabled: Bool) async { + self.logger.info("talk enabled=\(enabled)") + if enabled { + TalkOverlayController.shared.present() + } else { + TalkOverlayController.shared.dismiss() + } + await TalkModeRuntime.shared.setEnabled(enabled) + } + + func updatePhase(_ phase: TalkModePhase) { + self.phase = phase + TalkOverlayController.shared.updatePhase(phase) + let effectivePhase = self.isPaused ? "paused" : phase.rawValue + Task { + await GatewayConnection.shared.talkMode( + enabled: AppStateStore.shared.talkEnabled, + phase: effectivePhase) + } + } + + func updateLevel(_ level: Double) { + TalkOverlayController.shared.updateLevel(level) + } + + func setPaused(_ paused: Bool) { + guard self.isPaused != paused else { return } + self.logger.info("talk paused=\(paused)") + self.isPaused = paused + TalkOverlayController.shared.updatePaused(paused) + let effectivePhase = paused ? "paused" : self.phase.rawValue + Task { + await GatewayConnection.shared.talkMode( + enabled: AppStateStore.shared.talkEnabled, + phase: effectivePhase) + } + Task { await TalkModeRuntime.shared.setPaused(paused) } + } + + func togglePaused() { + self.setPaused(!self.isPaused) + } + + func stopSpeaking(reason: TalkStopReason = .userTap) { + Task { await TalkModeRuntime.shared.stopSpeaking(reason: reason) } + } + + func exitTalkMode() { + Task { await AppStateStore.shared.setTalkEnabled(false) } + } +} + +enum TalkStopReason { + case userTap + case speech + case manual +} diff --git a/apps/macos/Sources/OpenClaw/TalkModeRuntime.swift b/apps/macos/Sources/OpenClaw/TalkModeRuntime.swift new file mode 100644 index 0000000000000000000000000000000000000000..3da2389bfe6b7054a7e524d184757a6fbd5d3c4e --- /dev/null +++ b/apps/macos/Sources/OpenClaw/TalkModeRuntime.swift @@ -0,0 +1,953 @@ +import AVFoundation +import OpenClawChatUI +import OpenClawKit +import Foundation +import OSLog +import Speech + +actor TalkModeRuntime { + static let shared = TalkModeRuntime() + + private let logger = Logger(subsystem: "ai.openclaw", category: "talk.runtime") + private let ttsLogger = Logger(subsystem: "ai.openclaw", category: "talk.tts") + private static let defaultModelIdFallback = "eleven_v3" + + private final class RMSMeter: @unchecked Sendable { + private let lock = NSLock() + private var latestRMS: Double = 0 + + func set(_ rms: Double) { + self.lock.lock() + self.latestRMS = rms + self.lock.unlock() + } + + func get() -> Double { + self.lock.lock() + let value = self.latestRMS + self.lock.unlock() + return value + } + } + + private var recognizer: SFSpeechRecognizer? + private var audioEngine: AVAudioEngine? + private var recognitionRequest: SFSpeechAudioBufferRecognitionRequest? + private var recognitionTask: SFSpeechRecognitionTask? + private var recognitionGeneration: Int = 0 + private var rmsTask: Task? + private let rmsMeter = RMSMeter() + + private var captureTask: Task? + private var silenceTask: Task? + private var phase: TalkModePhase = .idle + private var isEnabled = false + private var isPaused = false + private var lifecycleGeneration: Int = 0 + + private var lastHeard: Date? + private var noiseFloorRMS: Double = 1e-4 + private var lastTranscript: String = "" + private var lastSpeechEnergyAt: Date? + + private var defaultVoiceId: String? + private var currentVoiceId: String? + private var defaultModelId: String? + private var currentModelId: String? + private var voiceOverrideActive = false + private var modelOverrideActive = false + private var defaultOutputFormat: String? + private var interruptOnSpeech: Bool = true + private var lastInterruptedAtSeconds: Double? + private var voiceAliases: [String: String] = [:] + private var lastSpokenText: String? + private var apiKey: String? + private var fallbackVoiceId: String? + private var lastPlaybackWasPCM: Bool = false + + private let silenceWindow: TimeInterval = 0.7 + private let minSpeechRMS: Double = 1e-3 + private let speechBoostFactor: Double = 6.0 + + // MARK: - Lifecycle + + func setEnabled(_ enabled: Bool) async { + guard enabled != self.isEnabled else { return } + self.isEnabled = enabled + self.lifecycleGeneration &+= 1 + if enabled { + await self.start() + } else { + await self.stop() + } + } + + func setPaused(_ paused: Bool) async { + guard paused != self.isPaused else { return } + self.isPaused = paused + await MainActor.run { TalkModeController.shared.updateLevel(0) } + + guard self.isEnabled else { return } + + if paused { + self.lastTranscript = "" + self.lastHeard = nil + self.lastSpeechEnergyAt = nil + await self.stopRecognition() + return + } + + if self.phase == .idle || self.phase == .listening { + await self.startRecognition() + self.phase = .listening + await MainActor.run { TalkModeController.shared.updatePhase(.listening) } + self.startSilenceMonitor() + } + } + + private func isCurrent(_ generation: Int) -> Bool { + generation == self.lifecycleGeneration && self.isEnabled + } + + private func start() async { + let gen = self.lifecycleGeneration + guard voiceWakeSupported else { return } + guard PermissionManager.voiceWakePermissionsGranted() else { + self.logger.debug("talk runtime not starting: permissions missing") + return + } + await self.reloadConfig() + guard self.isCurrent(gen) else { return } + if self.isPaused { + self.phase = .idle + await MainActor.run { + TalkModeController.shared.updateLevel(0) + TalkModeController.shared.updatePhase(.idle) + } + return + } + await self.startRecognition() + guard self.isCurrent(gen) else { return } + self.phase = .listening + await MainActor.run { TalkModeController.shared.updatePhase(.listening) } + self.startSilenceMonitor() + } + + private func stop() async { + self.captureTask?.cancel() + self.captureTask = nil + self.silenceTask?.cancel() + self.silenceTask = nil + + // Stop audio before changing phase (stopSpeaking is gated on .speaking). + await self.stopSpeaking(reason: .manual) + + self.lastTranscript = "" + self.lastHeard = nil + self.lastSpeechEnergyAt = nil + self.phase = .idle + await self.stopRecognition() + await MainActor.run { + TalkModeController.shared.updateLevel(0) + TalkModeController.shared.updatePhase(.idle) + } + } + + // MARK: - Speech recognition + + private struct RecognitionUpdate { + let transcript: String? + let hasConfidence: Bool + let isFinal: Bool + let errorDescription: String? + let generation: Int + } + + private func startRecognition() async { + await self.stopRecognition() + self.recognitionGeneration &+= 1 + let generation = self.recognitionGeneration + + let locale = await MainActor.run { AppStateStore.shared.voiceWakeLocaleID } + self.recognizer = SFSpeechRecognizer(locale: Locale(identifier: locale)) + guard let recognizer, recognizer.isAvailable else { + self.logger.error("talk recognizer unavailable") + return + } + + self.recognitionRequest = SFSpeechAudioBufferRecognitionRequest() + self.recognitionRequest?.shouldReportPartialResults = true + guard let request = self.recognitionRequest else { return } + + if self.audioEngine == nil { + self.audioEngine = AVAudioEngine() + } + guard let audioEngine = self.audioEngine else { return } + + let input = audioEngine.inputNode + let format = input.outputFormat(forBus: 0) + input.removeTap(onBus: 0) + let meter = self.rmsMeter + input.installTap(onBus: 0, bufferSize: 2048, format: format) { [weak request, meter] buffer, _ in + request?.append(buffer) + if let rms = Self.rmsLevel(buffer: buffer) { + meter.set(rms) + } + } + + audioEngine.prepare() + do { + try audioEngine.start() + } catch { + self.logger.error("talk audio engine start failed: \(error.localizedDescription, privacy: .public)") + return + } + + self.startRMSTicker(meter: meter) + + self.recognitionTask = recognizer.recognitionTask(with: request) { [weak self, generation] result, error in + guard let self else { return } + let segments = result?.bestTranscription.segments ?? [] + let transcript = result?.bestTranscription.formattedString + let update = RecognitionUpdate( + transcript: transcript, + hasConfidence: segments.contains { $0.confidence > 0.6 }, + isFinal: result?.isFinal ?? false, + errorDescription: error?.localizedDescription, + generation: generation) + Task { await self.handleRecognition(update) } + } + } + + private func stopRecognition() async { + self.recognitionGeneration &+= 1 + self.recognitionTask?.cancel() + self.recognitionTask = nil + self.recognitionRequest?.endAudio() + self.recognitionRequest = nil + self.audioEngine?.inputNode.removeTap(onBus: 0) + self.audioEngine?.stop() + self.audioEngine = nil + self.recognizer = nil + self.rmsTask?.cancel() + self.rmsTask = nil + } + + private func startRMSTicker(meter: RMSMeter) { + self.rmsTask?.cancel() + self.rmsTask = Task { [weak self, meter] in + while let self { + try? await Task.sleep(nanoseconds: 50_000_000) + if Task.isCancelled { return } + await self.noteAudioLevel(rms: meter.get()) + } + } + } + + private func handleRecognition(_ update: RecognitionUpdate) async { + guard update.generation == self.recognitionGeneration else { return } + guard !self.isPaused else { return } + if let errorDescription = update.errorDescription { + self.logger.debug("talk recognition error: \(errorDescription, privacy: .public)") + } + guard let transcript = update.transcript else { return } + + let trimmed = transcript.trimmingCharacters(in: .whitespacesAndNewlines) + if self.phase == .speaking, self.interruptOnSpeech { + if await self.shouldInterrupt(transcript: trimmed, hasConfidence: update.hasConfidence) { + await self.stopSpeaking(reason: .speech) + self.lastTranscript = "" + self.lastHeard = nil + await self.startListening() + } + return + } + + guard self.phase == .listening else { return } + + if !trimmed.isEmpty { + self.lastTranscript = trimmed + self.lastHeard = Date() + } + + if update.isFinal { + self.lastTranscript = trimmed + } + } + + // MARK: - Silence handling + + private func startSilenceMonitor() { + self.silenceTask?.cancel() + self.silenceTask = Task { [weak self] in + await self?.silenceLoop() + } + } + + private func silenceLoop() async { + while self.isEnabled { + try? await Task.sleep(nanoseconds: 200_000_000) + await self.checkSilence() + } + } + + private func checkSilence() async { + guard !self.isPaused else { return } + guard self.phase == .listening else { return } + let transcript = self.lastTranscript.trimmingCharacters(in: .whitespacesAndNewlines) + guard !transcript.isEmpty else { return } + guard let lastHeard else { return } + let elapsed = Date().timeIntervalSince(lastHeard) + guard elapsed >= self.silenceWindow else { return } + await self.finalizeTranscript(transcript) + } + + private func startListening() async { + self.phase = .listening + self.lastTranscript = "" + self.lastHeard = nil + await MainActor.run { + TalkModeController.shared.updatePhase(.listening) + TalkModeController.shared.updateLevel(0) + } + } + + private func finalizeTranscript(_ text: String) async { + self.lastTranscript = "" + self.lastHeard = nil + self.phase = .thinking + await MainActor.run { TalkModeController.shared.updatePhase(.thinking) } + await self.stopRecognition() + await self.sendAndSpeak(text) + } + + // MARK: - Gateway + TTS + + private func sendAndSpeak(_ transcript: String) async { + let gen = self.lifecycleGeneration + await self.reloadConfig() + guard self.isCurrent(gen) else { return } + let prompt = self.buildPrompt(transcript: transcript) + let activeSessionKey = await MainActor.run { WebChatManager.shared.activeSessionKey } + let sessionKey: String = if let activeSessionKey { + activeSessionKey + } else { + await GatewayConnection.shared.mainSessionKey() + } + let runId = UUID().uuidString + let startedAt = Date().timeIntervalSince1970 + self.logger.info( + "talk send start runId=\(runId, privacy: .public) " + + "session=\(sessionKey, privacy: .public) " + + "chars=\(prompt.count, privacy: .public)") + + do { + let response = try await GatewayConnection.shared.chatSend( + sessionKey: sessionKey, + message: prompt, + thinking: "low", + idempotencyKey: runId, + attachments: []) + guard self.isCurrent(gen) else { return } + self.logger.info( + "talk chat.send ok runId=\(response.runId, privacy: .public) " + + "session=\(sessionKey, privacy: .public)") + + guard let assistantText = await self.waitForAssistantText( + sessionKey: sessionKey, + since: startedAt, + timeoutSeconds: 45) + else { + self.logger.warning("talk assistant text missing after timeout") + await self.startListening() + await self.startRecognition() + return + } + guard self.isCurrent(gen) else { return } + + self.logger.info("talk assistant text len=\(assistantText.count, privacy: .public)") + await self.playAssistant(text: assistantText) + guard self.isCurrent(gen) else { return } + await self.resumeListeningIfNeeded() + return + } catch { + self.logger.error("talk chat.send failed: \(error.localizedDescription, privacy: .public)") + await self.resumeListeningIfNeeded() + return + } + } + + private func resumeListeningIfNeeded() async { + if self.isPaused { + self.lastTranscript = "" + self.lastHeard = nil + self.lastSpeechEnergyAt = nil + await MainActor.run { + TalkModeController.shared.updateLevel(0) + } + return + } + await self.startListening() + await self.startRecognition() + } + + private func buildPrompt(transcript: String) -> String { + let interrupted = self.lastInterruptedAtSeconds + self.lastInterruptedAtSeconds = nil + return TalkPromptBuilder.build(transcript: transcript, interruptedAtSeconds: interrupted) + } + + private func waitForAssistantText( + sessionKey: String, + since: Double, + timeoutSeconds: Int) async -> String? + { + let deadline = Date().addingTimeInterval(TimeInterval(timeoutSeconds)) + while Date() < deadline { + if let text = await self.latestAssistantText(sessionKey: sessionKey, since: since) { + return text + } + try? await Task.sleep(nanoseconds: 300_000_000) + } + return nil + } + + private func latestAssistantText(sessionKey: String, since: Double? = nil) async -> String? { + do { + let history = try await GatewayConnection.shared.chatHistory(sessionKey: sessionKey) + let messages = history.messages ?? [] + let decoded: [OpenClawChatMessage] = messages.compactMap { item in + guard let data = try? JSONEncoder().encode(item) else { return nil } + return try? JSONDecoder().decode(OpenClawChatMessage.self, from: data) + } + let assistant = decoded.last { message in + guard message.role == "assistant" else { return false } + guard let since else { return true } + guard let timestamp = message.timestamp else { return false } + return TalkHistoryTimestamp.isAfter(timestamp, sinceSeconds: since) + } + guard let assistant else { return nil } + let text = assistant.content.compactMap(\.text).joined(separator: "\n") + let trimmed = text.trimmingCharacters(in: CharacterSet.whitespacesAndNewlines) + return trimmed.isEmpty ? nil : trimmed + } catch { + self.logger.error("talk history fetch failed: \(error.localizedDescription, privacy: .public)") + return nil + } + } + + private func playAssistant(text: String) async { + guard let input = await self.preparePlaybackInput(text: text) else { return } + do { + if let apiKey = input.apiKey, !apiKey.isEmpty, let voiceId = input.voiceId { + try await self.playElevenLabs(input: input, apiKey: apiKey, voiceId: voiceId) + } else { + try await self.playSystemVoice(input: input) + } + } catch { + self.ttsLogger + .error( + "talk TTS failed: \(error.localizedDescription, privacy: .public); " + + "falling back to system voice") + do { + try await self.playSystemVoice(input: input) + } catch { + self.ttsLogger.error("talk system voice failed: \(error.localizedDescription, privacy: .public)") + } + } + + if self.phase == .speaking { + self.phase = .thinking + await MainActor.run { TalkModeController.shared.updatePhase(.thinking) } + } + } + + private struct TalkPlaybackInput { + let generation: Int + let cleanedText: String + let directive: TalkDirective? + let apiKey: String? + let voiceId: String? + let language: String? + let synthTimeoutSeconds: Double + } + + private func preparePlaybackInput(text: String) async -> TalkPlaybackInput? { + let gen = self.lifecycleGeneration + let parse = TalkDirectiveParser.parse(text) + let directive = parse.directive + let cleaned = parse.stripped.trimmingCharacters(in: .whitespacesAndNewlines) + guard !cleaned.isEmpty else { return nil } + guard self.isCurrent(gen) else { return nil } + + if !parse.unknownKeys.isEmpty { + self.logger + .warning( + "talk directive ignored keys: " + + "\(parse.unknownKeys.joined(separator: ","), privacy: .public)") + } + + let requestedVoice = directive?.voiceId?.trimmingCharacters(in: .whitespacesAndNewlines) + let resolvedVoice = self.resolveVoiceAlias(requestedVoice) + if let requestedVoice, !requestedVoice.isEmpty, resolvedVoice == nil { + self.logger.warning("talk unknown voice alias \(requestedVoice, privacy: .public)") + } + if let voice = resolvedVoice { + if directive?.once == true { + self.logger.info("talk voice override (once) voiceId=\(voice, privacy: .public)") + } else { + self.currentVoiceId = voice + self.voiceOverrideActive = true + self.logger.info("talk voice override voiceId=\(voice, privacy: .public)") + } + } + + if let model = directive?.modelId { + if directive?.once == true { + self.logger.info("talk model override (once) modelId=\(model, privacy: .public)") + } else { + self.currentModelId = model + self.modelOverrideActive = true + } + } + + let apiKey = self.apiKey?.trimmingCharacters(in: .whitespacesAndNewlines) + let preferredVoice = + resolvedVoice ?? + self.currentVoiceId ?? + self.defaultVoiceId + + let language = ElevenLabsTTSClient.validatedLanguage(directive?.language) + + let voiceId: String? = if let apiKey, !apiKey.isEmpty { + await self.resolveVoiceId(preferred: preferredVoice, apiKey: apiKey) + } else { + nil + } + + if apiKey?.isEmpty != false { + self.ttsLogger.warning("talk missing ELEVENLABS_API_KEY; falling back to system voice") + } else if voiceId == nil { + self.ttsLogger.warning("talk missing voiceId; falling back to system voice") + } else if let voiceId { + self.ttsLogger + .info( + "talk TTS request voiceId=\(voiceId, privacy: .public) " + + "chars=\(cleaned.count, privacy: .public)") + } + self.lastSpokenText = cleaned + + let synthTimeoutSeconds = max(20.0, min(90.0, Double(cleaned.count) * 0.12)) + + guard self.isCurrent(gen) else { return nil } + + return TalkPlaybackInput( + generation: gen, + cleanedText: cleaned, + directive: directive, + apiKey: apiKey, + voiceId: voiceId, + language: language, + synthTimeoutSeconds: synthTimeoutSeconds) + } + + private func playElevenLabs(input: TalkPlaybackInput, apiKey: String, voiceId: String) async throws { + let desiredOutputFormat = input.directive?.outputFormat ?? self.defaultOutputFormat ?? "pcm_44100" + let outputFormat = ElevenLabsTTSClient.validatedOutputFormat(desiredOutputFormat) + if outputFormat == nil, !desiredOutputFormat.isEmpty { + self.logger + .warning( + "talk output_format unsupported for local playback: " + + "\(desiredOutputFormat, privacy: .public)") + } + + let modelId = input.directive?.modelId ?? self.currentModelId ?? self.defaultModelId + func makeRequest(outputFormat: String?) -> ElevenLabsTTSRequest { + ElevenLabsTTSRequest( + text: input.cleanedText, + modelId: modelId, + outputFormat: outputFormat, + speed: TalkTTSValidation.resolveSpeed( + speed: input.directive?.speed, + rateWPM: input.directive?.rateWPM), + stability: TalkTTSValidation.validatedStability( + input.directive?.stability, + modelId: modelId), + similarity: TalkTTSValidation.validatedUnit(input.directive?.similarity), + style: TalkTTSValidation.validatedUnit(input.directive?.style), + speakerBoost: input.directive?.speakerBoost, + seed: TalkTTSValidation.validatedSeed(input.directive?.seed), + normalize: ElevenLabsTTSClient.validatedNormalize(input.directive?.normalize), + language: input.language, + latencyTier: TalkTTSValidation.validatedLatencyTier(input.directive?.latencyTier)) + } + + let request = makeRequest(outputFormat: outputFormat) + self.ttsLogger.info("talk TTS synth timeout=\(input.synthTimeoutSeconds, privacy: .public)s") + let client = ElevenLabsTTSClient(apiKey: apiKey) + let stream = client.streamSynthesize(voiceId: voiceId, request: request) + guard self.isCurrent(input.generation) else { return } + + if self.interruptOnSpeech { + guard await self.prepareForPlayback(generation: input.generation) else { return } + } + + await MainActor.run { TalkModeController.shared.updatePhase(.speaking) } + self.phase = .speaking + + let result = await self.playRemoteStream( + client: client, + voiceId: voiceId, + outputFormat: outputFormat, + makeRequest: makeRequest, + stream: stream) + self.ttsLogger + .info( + "talk audio result finished=\(result.finished, privacy: .public) " + + "interruptedAt=\(String(describing: result.interruptedAt), privacy: .public)") + if !result.finished, result.interruptedAt == nil { + throw NSError(domain: "StreamingAudioPlayer", code: 1, userInfo: [ + NSLocalizedDescriptionKey: "audio playback failed", + ]) + } + if !result.finished, let interruptedAt = result.interruptedAt, self.phase == .speaking { + if self.interruptOnSpeech { + self.lastInterruptedAtSeconds = interruptedAt + } + } + } + + private func playRemoteStream( + client: ElevenLabsTTSClient, + voiceId: String, + outputFormat: String?, + makeRequest: (String?) -> ElevenLabsTTSRequest, + stream: AsyncThrowingStream) async -> StreamingPlaybackResult + { + let sampleRate = TalkTTSValidation.pcmSampleRate(from: outputFormat) + if let sampleRate { + self.lastPlaybackWasPCM = true + let result = await self.playPCM(stream: stream, sampleRate: sampleRate) + if result.finished || result.interruptedAt != nil { + return result + } + let mp3Format = ElevenLabsTTSClient.validatedOutputFormat("mp3_44100") + self.ttsLogger.warning("talk pcm playback failed; retrying mp3") + self.lastPlaybackWasPCM = false + let mp3Stream = client.streamSynthesize( + voiceId: voiceId, + request: makeRequest(mp3Format)) + return await self.playMP3(stream: mp3Stream) + } + self.lastPlaybackWasPCM = false + return await self.playMP3(stream: stream) + } + + private func playSystemVoice(input: TalkPlaybackInput) async throws { + self.ttsLogger.info("talk system voice start chars=\(input.cleanedText.count, privacy: .public)") + if self.interruptOnSpeech { + guard await self.prepareForPlayback(generation: input.generation) else { return } + } + await MainActor.run { TalkModeController.shared.updatePhase(.speaking) } + self.phase = .speaking + await TalkSystemSpeechSynthesizer.shared.stop() + try await TalkSystemSpeechSynthesizer.shared.speak( + text: input.cleanedText, + language: input.language) + self.ttsLogger.info("talk system voice done") + } + + private func prepareForPlayback(generation: Int) async -> Bool { + await self.startRecognition() + return self.isCurrent(generation) + } + + private func resolveVoiceId(preferred: String?, apiKey: String) async -> String? { + let trimmed = preferred?.trimmingCharacters(in: .whitespacesAndNewlines) ?? "" + if !trimmed.isEmpty { + if let resolved = self.resolveVoiceAlias(trimmed) { return resolved } + self.ttsLogger.warning("talk unknown voice alias \(trimmed, privacy: .public)") + } + if let fallbackVoiceId { return fallbackVoiceId } + + do { + let voices = try await ElevenLabsTTSClient(apiKey: apiKey).listVoices() + guard let first = voices.first else { + self.ttsLogger.error("elevenlabs voices list empty") + return nil + } + self.fallbackVoiceId = first.voiceId + if self.defaultVoiceId == nil { + self.defaultVoiceId = first.voiceId + } + if !self.voiceOverrideActive { + self.currentVoiceId = first.voiceId + } + let name = first.name ?? "unknown" + self.ttsLogger + .info("talk default voice selected \(name, privacy: .public) (\(first.voiceId, privacy: .public))") + return first.voiceId + } catch { + self.ttsLogger.error("elevenlabs list voices failed: \(error.localizedDescription, privacy: .public)") + return nil + } + } + + private func resolveVoiceAlias(_ value: String?) -> String? { + let trimmed = (value ?? "").trimmingCharacters(in: .whitespacesAndNewlines) + guard !trimmed.isEmpty else { return nil } + let normalized = trimmed.lowercased() + if let mapped = self.voiceAliases[normalized] { return mapped } + if self.voiceAliases.values.contains(where: { $0.caseInsensitiveCompare(trimmed) == .orderedSame }) { + return trimmed + } + return Self.isLikelyVoiceId(trimmed) ? trimmed : nil + } + + private static func isLikelyVoiceId(_ value: String) -> Bool { + guard value.count >= 10 else { return false } + return value.allSatisfy { $0.isLetter || $0.isNumber || $0 == "-" || $0 == "_" } + } + + func stopSpeaking(reason: TalkStopReason) async { + let usePCM = self.lastPlaybackWasPCM + let interruptedAt = usePCM ? await self.stopPCM() : await self.stopMP3() + _ = usePCM ? await self.stopMP3() : await self.stopPCM() + await TalkSystemSpeechSynthesizer.shared.stop() + guard self.phase == .speaking else { return } + if reason == .speech, let interruptedAt { + self.lastInterruptedAtSeconds = interruptedAt + } + if reason == .manual { + return + } + if reason == .speech || reason == .userTap { + await self.startListening() + return + } + self.phase = .thinking + await MainActor.run { TalkModeController.shared.updatePhase(.thinking) } + } +} + +extension TalkModeRuntime { + // MARK: - Audio playback (MainActor helpers) + + @MainActor + private func playPCM( + stream: AsyncThrowingStream, + sampleRate: Double) async -> StreamingPlaybackResult + { + await PCMStreamingAudioPlayer.shared.play(stream: stream, sampleRate: sampleRate) + } + + @MainActor + private func playMP3(stream: AsyncThrowingStream) async -> StreamingPlaybackResult { + await StreamingAudioPlayer.shared.play(stream: stream) + } + + @MainActor + private func stopPCM() -> Double? { + PCMStreamingAudioPlayer.shared.stop() + } + + @MainActor + private func stopMP3() -> Double? { + StreamingAudioPlayer.shared.stop() + } + + // MARK: - Config + + private func reloadConfig() async { + let cfg = await self.fetchTalkConfig() + self.defaultVoiceId = cfg.voiceId + self.voiceAliases = cfg.voiceAliases + if !self.voiceOverrideActive { + self.currentVoiceId = cfg.voiceId + } + self.defaultModelId = cfg.modelId + if !self.modelOverrideActive { + self.currentModelId = cfg.modelId + } + self.defaultOutputFormat = cfg.outputFormat + self.interruptOnSpeech = cfg.interruptOnSpeech + self.apiKey = cfg.apiKey + let hasApiKey = (cfg.apiKey?.isEmpty == false) + let voiceLabel = (cfg.voiceId?.isEmpty == false) ? cfg.voiceId! : "none" + let modelLabel = (cfg.modelId?.isEmpty == false) ? cfg.modelId! : "none" + self.logger + .info( + "talk config voiceId=\(voiceLabel, privacy: .public) " + + "modelId=\(modelLabel, privacy: .public) " + + "apiKey=\(hasApiKey, privacy: .public) " + + "interrupt=\(cfg.interruptOnSpeech, privacy: .public)") + } + + private struct TalkRuntimeConfig { + let voiceId: String? + let voiceAliases: [String: String] + let modelId: String? + let outputFormat: String? + let interruptOnSpeech: Bool + let apiKey: String? + } + + private func fetchTalkConfig() async -> TalkRuntimeConfig { + let env = ProcessInfo.processInfo.environment + let envVoice = env["ELEVENLABS_VOICE_ID"]?.trimmingCharacters(in: .whitespacesAndNewlines) + let sagVoice = env["SAG_VOICE_ID"]?.trimmingCharacters(in: .whitespacesAndNewlines) + let envApiKey = env["ELEVENLABS_API_KEY"]?.trimmingCharacters(in: .whitespacesAndNewlines) + + do { + let snap: ConfigSnapshot = try await GatewayConnection.shared.requestDecoded( + method: .configGet, + params: nil, + timeoutMs: 8000) + let talk = snap.config?["talk"]?.dictionaryValue + let ui = snap.config?["ui"]?.dictionaryValue + let rawSeam = ui?["seamColor"]?.stringValue?.trimmingCharacters(in: .whitespacesAndNewlines) ?? "" + await MainActor.run { + AppStateStore.shared.seamColorHex = rawSeam.isEmpty ? nil : rawSeam + } + let voice = talk?["voiceId"]?.stringValue + let rawAliases = talk?["voiceAliases"]?.dictionaryValue + let resolvedAliases: [String: String] = + rawAliases?.reduce(into: [:]) { acc, entry in + let key = entry.key.trimmingCharacters(in: .whitespacesAndNewlines).lowercased() + let value = entry.value.stringValue?.trimmingCharacters(in: .whitespacesAndNewlines) ?? "" + guard !key.isEmpty, !value.isEmpty else { return } + acc[key] = value + } ?? [:] + let model = talk?["modelId"]?.stringValue?.trimmingCharacters(in: .whitespacesAndNewlines) + let resolvedModel = (model?.isEmpty == false) ? model! : Self.defaultModelIdFallback + let outputFormat = talk?["outputFormat"]?.stringValue + let interrupt = talk?["interruptOnSpeech"]?.boolValue + let apiKey = talk?["apiKey"]?.stringValue + let resolvedVoice = + (voice?.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty == false ? voice : nil) ?? + (envVoice?.isEmpty == false ? envVoice : nil) ?? + (sagVoice?.isEmpty == false ? sagVoice : nil) + let resolvedApiKey = + (envApiKey?.isEmpty == false ? envApiKey : nil) ?? + (apiKey?.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty == false ? apiKey : nil) + return TalkRuntimeConfig( + voiceId: resolvedVoice, + voiceAliases: resolvedAliases, + modelId: resolvedModel, + outputFormat: outputFormat, + interruptOnSpeech: interrupt ?? true, + apiKey: resolvedApiKey) + } catch { + let resolvedVoice = + (envVoice?.isEmpty == false ? envVoice : nil) ?? + (sagVoice?.isEmpty == false ? sagVoice : nil) + let resolvedApiKey = envApiKey?.isEmpty == false ? envApiKey : nil + return TalkRuntimeConfig( + voiceId: resolvedVoice, + voiceAliases: [:], + modelId: Self.defaultModelIdFallback, + outputFormat: nil, + interruptOnSpeech: true, + apiKey: resolvedApiKey) + } + } + + // MARK: - Audio level handling + + private func noteAudioLevel(rms: Double) async { + if self.phase != .listening, self.phase != .speaking { return } + let alpha: Double = rms < self.noiseFloorRMS ? 0.08 : 0.01 + self.noiseFloorRMS = max(1e-7, self.noiseFloorRMS + (rms - self.noiseFloorRMS) * alpha) + + let threshold = max(self.minSpeechRMS, self.noiseFloorRMS * self.speechBoostFactor) + if rms >= threshold { + let now = Date() + self.lastHeard = now + self.lastSpeechEnergyAt = now + } + + if self.phase == .listening { + let clamped = min(1.0, max(0.0, rms / max(self.minSpeechRMS, threshold))) + await MainActor.run { TalkModeController.shared.updateLevel(clamped) } + } + } + + private static func rmsLevel(buffer: AVAudioPCMBuffer) -> Double? { + guard let channelData = buffer.floatChannelData?.pointee else { return nil } + let frameCount = Int(buffer.frameLength) + guard frameCount > 0 else { return nil } + var sum: Double = 0 + for i in 0.. Bool { + let trimmed = transcript.trimmingCharacters(in: .whitespacesAndNewlines) + guard trimmed.count >= 3 else { return false } + if self.isLikelyEcho(of: trimmed) { return false } + let now = Date() + if let lastSpeechEnergyAt, now.timeIntervalSince(lastSpeechEnergyAt) > 0.35 { + return false + } + return hasConfidence + } + + private func isLikelyEcho(of transcript: String) -> Bool { + guard let spoken = self.lastSpokenText?.lowercased(), !spoken.isEmpty else { return false } + let probe = transcript.lowercased() + if probe.count < 6 { + return spoken.contains(probe) + } + return spoken.contains(probe) + } + + private static func resolveSpeed(speed: Double?, rateWPM: Int?, logger: Logger) -> Double? { + if let rateWPM, rateWPM > 0 { + let resolved = Double(rateWPM) / 175.0 + if resolved <= 0.5 || resolved >= 2.0 { + logger.warning("talk rateWPM out of range: \(rateWPM, privacy: .public)") + return nil + } + return resolved + } + if let speed { + if speed <= 0.5 || speed >= 2.0 { + logger.warning("talk speed out of range: \(speed, privacy: .public)") + return nil + } + return speed + } + return nil + } + + private static func validatedUnit(_ value: Double?, name: String, logger: Logger) -> Double? { + guard let value else { return nil } + if value < 0 || value > 1 { + logger.warning("talk \(name, privacy: .public) out of range: \(value, privacy: .public)") + return nil + } + return value + } + + private static func validatedSeed(_ value: Int?, logger: Logger) -> UInt32? { + guard let value else { return nil } + if value < 0 || value > 4_294_967_295 { + logger.warning("talk seed out of range: \(value, privacy: .public)") + return nil + } + return UInt32(value) + } + + private static func validatedNormalize(_ value: String?, logger: Logger) -> String? { + guard let value else { return nil } + let normalized = value.trimmingCharacters(in: .whitespacesAndNewlines).lowercased() + guard ["auto", "on", "off"].contains(normalized) else { + logger.warning("talk normalize invalid: \(normalized, privacy: .public)") + return nil + } + return normalized + } +} diff --git a/apps/macos/Sources/OpenClaw/TalkModeTypes.swift b/apps/macos/Sources/OpenClaw/TalkModeTypes.swift new file mode 100644 index 0000000000000000000000000000000000000000..3ae978255f4ccce8aebe5c2c083f357cf47f08fb --- /dev/null +++ b/apps/macos/Sources/OpenClaw/TalkModeTypes.swift @@ -0,0 +1,8 @@ +import Foundation + +enum TalkModePhase: String { + case idle + case listening + case thinking + case speaking +} diff --git a/apps/macos/Sources/OpenClaw/TalkOverlay.swift b/apps/macos/Sources/OpenClaw/TalkOverlay.swift new file mode 100644 index 0000000000000000000000000000000000000000..27e5dedc110905540880455bab3d21e7389bad1f --- /dev/null +++ b/apps/macos/Sources/OpenClaw/TalkOverlay.swift @@ -0,0 +1,146 @@ +import AppKit +import Observation +import OSLog +import SwiftUI + +@MainActor +@Observable +final class TalkOverlayController { + static let shared = TalkOverlayController() + static let overlaySize: CGFloat = 440 + static let orbSize: CGFloat = 96 + static let orbPadding: CGFloat = 12 + static let orbHitSlop: CGFloat = 10 + + private let logger = Logger(subsystem: "ai.openclaw", category: "talk.overlay") + + struct Model { + var isVisible: Bool = false + var phase: TalkModePhase = .idle + var isPaused: Bool = false + var level: Double = 0 + } + + var model = Model() + private var window: NSPanel? + private var hostingView: NSHostingView? + private let screenInset: CGFloat = 0 + + func present() { + self.ensureWindow() + self.hostingView?.rootView = TalkOverlayView(controller: self) + let target = self.targetFrame() + + guard let window else { return } + if !self.model.isVisible { + self.model.isVisible = true + let start = target.offsetBy(dx: 0, dy: -6) + window.setFrame(start, display: true) + window.alphaValue = 0 + window.orderFrontRegardless() + NSAnimationContext.runAnimationGroup { context in + context.duration = 0.18 + context.timingFunction = CAMediaTimingFunction(name: .easeOut) + window.animator().setFrame(target, display: true) + window.animator().alphaValue = 1 + } + } else { + window.setFrame(target, display: true) + window.orderFrontRegardless() + } + } + + func dismiss() { + guard let window else { + self.model.isVisible = false + return + } + + let target = window.frame.offsetBy(dx: 6, dy: 6) + NSAnimationContext.runAnimationGroup { context in + context.duration = 0.16 + context.timingFunction = CAMediaTimingFunction(name: .easeOut) + window.animator().setFrame(target, display: true) + window.animator().alphaValue = 0 + } completionHandler: { + Task { @MainActor in + window.orderOut(nil) + self.model.isVisible = false + } + } + } + + func updatePhase(_ phase: TalkModePhase) { + guard self.model.phase != phase else { return } + self.logger.info("talk overlay phase=\(phase.rawValue, privacy: .public)") + self.model.phase = phase + } + + func updatePaused(_ paused: Bool) { + guard self.model.isPaused != paused else { return } + self.logger.info("talk overlay paused=\(paused)") + self.model.isPaused = paused + } + + func updateLevel(_ level: Double) { + guard self.model.isVisible else { return } + self.model.level = max(0, min(1, level)) + } + + func currentWindowOrigin() -> CGPoint? { + self.window?.frame.origin + } + + func setWindowOrigin(_ origin: CGPoint) { + guard let window else { return } + window.setFrameOrigin(origin) + } + + // MARK: - Private + + private func ensureWindow() { + if self.window != nil { return } + let panel = NSPanel( + contentRect: NSRect(x: 0, y: 0, width: Self.overlaySize, height: Self.overlaySize), + styleMask: [.nonactivatingPanel, .borderless], + backing: .buffered, + defer: false) + panel.isOpaque = false + panel.backgroundColor = .clear + panel.hasShadow = false + panel.level = NSWindow.Level(rawValue: NSWindow.Level.popUpMenu.rawValue - 4) + panel.collectionBehavior = [.canJoinAllSpaces, .fullScreenAuxiliary, .transient] + panel.hidesOnDeactivate = false + panel.isMovable = false + panel.acceptsMouseMovedEvents = true + panel.isFloatingPanel = true + panel.becomesKeyOnlyIfNeeded = true + panel.titleVisibility = .hidden + panel.titlebarAppearsTransparent = true + + let host = TalkOverlayHostingView(rootView: TalkOverlayView(controller: self)) + host.translatesAutoresizingMaskIntoConstraints = false + panel.contentView = host + self.hostingView = host + self.window = panel + } + + private func targetFrame() -> NSRect { + let screen = self.window?.screen + ?? NSScreen.main + ?? NSScreen.screens.first + guard let screen else { return .zero } + let size = NSSize(width: Self.overlaySize, height: Self.overlaySize) + let visible = screen.visibleFrame + let origin = CGPoint( + x: visible.maxX - size.width - self.screenInset, + y: visible.maxY - size.height - self.screenInset) + return NSRect(origin: origin, size: size) + } +} + +private final class TalkOverlayHostingView: NSHostingView { + override func acceptsFirstMouse(for event: NSEvent?) -> Bool { + true + } +} diff --git a/apps/macos/Sources/OpenClaw/TalkOverlayView.swift b/apps/macos/Sources/OpenClaw/TalkOverlayView.swift new file mode 100644 index 0000000000000000000000000000000000000000..a24ba174374814638de5ef36daa903e2ecda986d --- /dev/null +++ b/apps/macos/Sources/OpenClaw/TalkOverlayView.swift @@ -0,0 +1,220 @@ +import AppKit +import SwiftUI + +struct TalkOverlayView: View { + var controller: TalkOverlayController + @State private var appState = AppStateStore.shared + @State private var hoveringWindow = false + + var body: some View { + ZStack(alignment: .topTrailing) { + let isPaused = self.controller.model.isPaused + Color.clear + TalkOrbView( + phase: self.controller.model.phase, + level: self.controller.model.level, + accent: self.seamColor, + isPaused: isPaused) + .frame(width: TalkOverlayController.orbSize, height: TalkOverlayController.orbSize) + .padding(.top, TalkOverlayController.orbPadding) + .padding(.trailing, TalkOverlayController.orbPadding) + .contentShape(Circle()) + .opacity(isPaused ? 0.55 : 1) + .background( + TalkOrbInteractionView( + onSingleClick: { TalkModeController.shared.togglePaused() }, + onDoubleClick: { TalkModeController.shared.stopSpeaking(reason: .userTap) }, + onDragStart: { TalkModeController.shared.setPaused(true) })) + .overlay(alignment: .topLeading) { + Button { + TalkModeController.shared.exitTalkMode() + } label: { + Image(systemName: "xmark") + .font(.system(size: 10, weight: .bold)) + .foregroundStyle(Color.white.opacity(0.95)) + .frame(width: 18, height: 18) + .background(Color.black.opacity(0.4)) + .clipShape(Circle()) + } + .buttonStyle(.plain) + .contentShape(Circle()) + .offset(x: -2, y: -2) + .opacity(self.hoveringWindow ? 1 : 0) + .animation(.easeOut(duration: 0.12), value: self.hoveringWindow) + } + .onHover { self.hoveringWindow = $0 } + } + .frame( + width: TalkOverlayController.overlaySize, + height: TalkOverlayController.overlaySize, + alignment: .topTrailing) + } + + private static let defaultSeamColor = Color(red: 79 / 255.0, green: 122 / 255.0, blue: 154 / 255.0) + + private var seamColor: Color { + Self.color(fromHex: self.appState.seamColorHex) ?? Self.defaultSeamColor + } + + private static func color(fromHex raw: String?) -> Color? { + let trimmed = (raw ?? "").trimmingCharacters(in: .whitespacesAndNewlines) + guard !trimmed.isEmpty else { return nil } + let hex = trimmed.hasPrefix("#") ? String(trimmed.dropFirst()) : trimmed + guard hex.count == 6, let value = Int(hex, radix: 16) else { return nil } + let r = Double((value >> 16) & 0xFF) / 255.0 + let g = Double((value >> 8) & 0xFF) / 255.0 + let b = Double(value & 0xFF) / 255.0 + return Color(red: r, green: g, blue: b) + } +} + +private struct TalkOrbInteractionView: NSViewRepresentable { + let onSingleClick: () -> Void + let onDoubleClick: () -> Void + let onDragStart: () -> Void + + func makeNSView(context: Context) -> NSView { + let view = OrbInteractionNSView() + view.onSingleClick = self.onSingleClick + view.onDoubleClick = self.onDoubleClick + view.onDragStart = self.onDragStart + view.wantsLayer = true + view.layer?.backgroundColor = NSColor.clear.cgColor + return view + } + + func updateNSView(_ nsView: NSView, context: Context) { + guard let view = nsView as? OrbInteractionNSView else { return } + view.onSingleClick = self.onSingleClick + view.onDoubleClick = self.onDoubleClick + view.onDragStart = self.onDragStart + } +} + +private final class OrbInteractionNSView: NSView { + var onSingleClick: (() -> Void)? + var onDoubleClick: (() -> Void)? + var onDragStart: (() -> Void)? + private var mouseDownEvent: NSEvent? + private var didDrag = false + private var suppressSingleClick = false + + override var acceptsFirstResponder: Bool { true } + override func acceptsFirstMouse(for event: NSEvent?) -> Bool { true } + + override func mouseDown(with event: NSEvent) { + self.mouseDownEvent = event + self.didDrag = false + self.suppressSingleClick = event.clickCount > 1 + if event.clickCount == 2 { + self.onDoubleClick?() + } + } + + override func mouseDragged(with event: NSEvent) { + guard let startEvent = self.mouseDownEvent else { return } + if !self.didDrag { + let dx = event.locationInWindow.x - startEvent.locationInWindow.x + let dy = event.locationInWindow.y - startEvent.locationInWindow.y + if abs(dx) + abs(dy) < 2 { return } + self.didDrag = true + self.onDragStart?() + self.window?.performDrag(with: startEvent) + } + } + + override func mouseUp(with event: NSEvent) { + if !self.didDrag, !self.suppressSingleClick { + self.onSingleClick?() + } + self.mouseDownEvent = nil + self.didDrag = false + self.suppressSingleClick = false + } +} + +private struct TalkOrbView: View { + let phase: TalkModePhase + let level: Double + let accent: Color + let isPaused: Bool + + var body: some View { + if self.isPaused { + Circle() + .fill(self.orbGradient) + .overlay(Circle().stroke(Color.white.opacity(0.35), lineWidth: 1)) + .shadow(color: Color.black.opacity(0.18), radius: 10, x: 0, y: 5) + } else { + TimelineView(.animation) { context in + let t = context.date.timeIntervalSinceReferenceDate + let listenScale = self.phase == .listening ? (1 + CGFloat(self.level) * 0.12) : 1 + let pulse = self.phase == .speaking ? (1 + 0.06 * sin(t * 6)) : 1 + + ZStack { + Circle() + .fill(self.orbGradient) + .overlay(Circle().stroke(Color.white.opacity(0.45), lineWidth: 1)) + .shadow(color: Color.black.opacity(0.22), radius: 10, x: 0, y: 5) + .scaleEffect(pulse * listenScale) + + TalkWaveRings(phase: self.phase, level: self.level, time: t, accent: self.accent) + + if self.phase == .thinking { + TalkOrbitArcs(time: t) + } + } + } + } + } + + private var orbGradient: RadialGradient { + RadialGradient( + colors: [Color.white, self.accent], + center: .topLeading, + startRadius: 4, + endRadius: 52) + } +} + +private struct TalkWaveRings: View { + let phase: TalkModePhase + let level: Double + let time: TimeInterval + let accent: Color + + var body: some View { + ZStack { + ForEach(0..<3, id: \.self) { idx in + let speed = self.phase == .speaking ? 1.4 : self.phase == .listening ? 0.9 : 0.6 + let progress = (time * speed + Double(idx) * 0.28).truncatingRemainder(dividingBy: 1) + let amplitude = self.phase == .speaking ? 0.95 : self.phase == .listening ? 0.5 + self + .level * 0.7 : 0.35 + let scale = 0.75 + progress * amplitude + (self.phase == .listening ? self.level * 0.15 : 0) + let alpha = self.phase == .speaking ? 0.72 : self.phase == .listening ? 0.58 + self.level * 0.28 : 0.4 + Circle() + .stroke(self.accent.opacity(alpha - progress * 0.3), lineWidth: 1.6) + .scaleEffect(scale) + .opacity(alpha - progress * 0.6) + } + } + } +} + +private struct TalkOrbitArcs: View { + let time: TimeInterval + + var body: some View { + ZStack { + Circle() + .trim(from: 0.08, to: 0.26) + .stroke(Color.white.opacity(0.88), style: StrokeStyle(lineWidth: 1.6, lineCap: .round)) + .rotationEffect(.degrees(self.time * 42)) + Circle() + .trim(from: 0.62, to: 0.86) + .stroke(Color.white.opacity(0.7), style: StrokeStyle(lineWidth: 1.4, lineCap: .round)) + .rotationEffect(.degrees(-self.time * 35)) + } + .scaleEffect(1.08) + } +} diff --git a/apps/macos/Sources/OpenClaw/TerminationSignalWatcher.swift b/apps/macos/Sources/OpenClaw/TerminationSignalWatcher.swift new file mode 100644 index 0000000000000000000000000000000000000000..add543c3ebe3bd17129001369104370ec11789f4 --- /dev/null +++ b/apps/macos/Sources/OpenClaw/TerminationSignalWatcher.swift @@ -0,0 +1,53 @@ +import AppKit +import Foundation +import OSLog + +@MainActor +final class TerminationSignalWatcher { + static let shared = TerminationSignalWatcher() + + private let logger = Logger(subsystem: "ai.openclaw", category: "lifecycle") + private var sources: [DispatchSourceSignal] = [] + private var terminationRequested = false + + func start() { + guard self.sources.isEmpty else { return } + self.install(SIGTERM) + self.install(SIGINT) + } + + func stop() { + for s in self.sources { + s.cancel() + } + self.sources.removeAll(keepingCapacity: false) + self.terminationRequested = false + } + + private func install(_ sig: Int32) { + // Make sure the default action doesn't kill the process before we can gracefully shut down. + signal(sig, SIG_IGN) + let source = DispatchSource.makeSignalSource(signal: sig, queue: .main) + source.setEventHandler { [weak self] in + self?.handle(sig) + } + source.resume() + self.sources.append(source) + } + + private func handle(_ sig: Int32) { + guard !self.terminationRequested else { return } + self.terminationRequested = true + + self.logger.info("received signal \(sig, privacy: .public); terminating") + // Ensure any pairing prompt can't accidentally approve during shutdown. + NodePairingApprovalPrompter.shared.stop() + DevicePairingApprovalPrompter.shared.stop() + NSApp.terminate(nil) + + // Safety net: don't hang forever if something blocks termination. + DispatchQueue.main.asyncAfter(deadline: .now() + 3) { + exit(0) + } + } +} diff --git a/apps/macos/Sources/OpenClaw/UsageCostData.swift b/apps/macos/Sources/OpenClaw/UsageCostData.swift new file mode 100644 index 0000000000000000000000000000000000000000..ca1fb5cc3e2aef04c18328e689d7f688427b21e8 --- /dev/null +++ b/apps/macos/Sources/OpenClaw/UsageCostData.swift @@ -0,0 +1,60 @@ +import Foundation + +struct GatewayCostUsageTotals: Codable { + let input: Int + let output: Int + let cacheRead: Int + let cacheWrite: Int + let totalTokens: Int + let totalCost: Double + let missingCostEntries: Int +} + +struct GatewayCostUsageDay: Codable { + let date: String + let input: Int + let output: Int + let cacheRead: Int + let cacheWrite: Int + let totalTokens: Int + let totalCost: Double + let missingCostEntries: Int +} + +struct GatewayCostUsageSummary: Codable { + let updatedAt: Double + let days: Int + let daily: [GatewayCostUsageDay] + let totals: GatewayCostUsageTotals +} + +enum CostUsageFormatting { + static func formatUsd(_ value: Double?) -> String? { + guard let value, value.isFinite else { return nil } + if value >= 1 { return String(format: "$%.2f", value) } + if value >= 0.01 { return String(format: "$%.2f", value) } + return String(format: "$%.4f", value) + } + + static func formatTokenCount(_ value: Int?) -> String? { + guard let value else { return nil } + let safe = max(0, value) + if safe >= 1_000_000 { return String(format: "%.1fm", Double(safe) / 1_000_000.0) } + if safe >= 1000 { return safe >= 10000 + ? String(format: "%.0fk", Double(safe) / 1000.0) + : String(format: "%.1fk", Double(safe) / 1000.0) + } + return String(safe) + } +} + +@MainActor +enum CostUsageLoader { + static func loadSummary() async throws -> GatewayCostUsageSummary { + let data = try await ControlChannel.shared.request( + method: "usage.cost", + params: nil, + timeoutMs: 7000) + return try JSONDecoder().decode(GatewayCostUsageSummary.self, from: data) + } +} diff --git a/apps/macos/Sources/OpenClaw/UsageData.swift b/apps/macos/Sources/OpenClaw/UsageData.swift new file mode 100644 index 0000000000000000000000000000000000000000..7800054c66c73edc7dc96fc0c35cbebe58661759 --- /dev/null +++ b/apps/macos/Sources/OpenClaw/UsageData.swift @@ -0,0 +1,104 @@ +import Foundation + +struct GatewayUsageWindow: Codable { + let label: String + let usedPercent: Double + let resetAt: Double? +} + +struct GatewayUsageProvider: Codable { + let provider: String + let displayName: String + let windows: [GatewayUsageWindow] + let plan: String? + let error: String? +} + +struct GatewayUsageSummary: Codable { + let updatedAt: Double + let providers: [GatewayUsageProvider] +} + +struct UsageRow: Identifiable { + let id: String + let providerId: String + let displayName: String + let plan: String? + let windowLabel: String? + let usedPercent: Double? + let resetAt: Date? + let error: String? + + var hasError: Bool { + if let error, !error.isEmpty { return true } + return false + } + + var titleText: String { + if let plan, !plan.isEmpty { return "\(self.displayName) (\(plan))" } + return self.displayName + } + + var remainingPercent: Int? { + guard let usedPercent, usedPercent.isFinite else { return nil } + let remaining = max(0, min(100, Int(round(100 - usedPercent)))) + return remaining + } + + func detailText(now: Date = .init()) -> String { + guard let remaining = self.remainingPercent else { return "No data" } + var parts = ["\(remaining)% left"] + if let windowLabel, !windowLabel.isEmpty { parts.append(windowLabel) } + if let resetAt { + let reset = UsageRow.formatResetRemaining(target: resetAt, now: now) + if let reset { parts.append("⏱\(reset)") } + } + return parts.joined(separator: " · ") + } + + private static func formatResetRemaining(target: Date, now: Date) -> String? { + let diff = target.timeIntervalSince(now) + if diff <= 0 { return "now" } + let minutes = Int(floor(diff / 60)) + if minutes < 60 { return "\(minutes)m" } + let hours = minutes / 60 + let mins = minutes % 60 + if hours < 24 { return mins > 0 ? "\(hours)h \(mins)m" : "\(hours)h" } + let days = hours / 24 + if days < 7 { return "\(days)d \(hours % 24)h" } + let formatter = DateFormatter() + formatter.dateFormat = "MMM d" + return formatter.string(from: target) + } +} + +extension GatewayUsageSummary { + func primaryRows() -> [UsageRow] { + self.providers.compactMap { provider in + guard let window = provider.windows.max(by: { $0.usedPercent < $1.usedPercent }) else { + return nil + } + + return UsageRow( + id: "\(provider.provider)-\(window.label)", + providerId: provider.provider, + displayName: provider.displayName, + plan: provider.plan, + windowLabel: window.label, + usedPercent: window.usedPercent, + resetAt: window.resetAt.map { Date(timeIntervalSince1970: $0 / 1000) }, + error: nil) + } + } +} + +@MainActor +enum UsageLoader { + static func loadSummary() async throws -> GatewayUsageSummary { + let data = try await ControlChannel.shared.request( + method: "usage.status", + params: nil, + timeoutMs: 5000) + return try JSONDecoder().decode(GatewayUsageSummary.self, from: data) + } +} diff --git a/apps/macos/Sources/OpenClaw/UsageMenuLabelView.swift b/apps/macos/Sources/OpenClaw/UsageMenuLabelView.swift new file mode 100644 index 0000000000000000000000000000000000000000..c7f95e476605d8566d8e543cffaaeb6bf6de45ab --- /dev/null +++ b/apps/macos/Sources/OpenClaw/UsageMenuLabelView.swift @@ -0,0 +1,59 @@ +import SwiftUI + +struct UsageMenuLabelView: View { + let row: UsageRow + let width: CGFloat + var showsChevron: Bool = false + @Environment(\.menuItemHighlighted) private var isHighlighted + private let paddingLeading: CGFloat = 22 + private let paddingTrailing: CGFloat = 14 + private let barHeight: CGFloat = 6 + + private var primaryTextColor: Color { + self.isHighlighted ? Color(nsColor: .selectedMenuItemTextColor) : .primary + } + + private var secondaryTextColor: Color { + self.isHighlighted ? Color(nsColor: .selectedMenuItemTextColor).opacity(0.85) : .secondary + } + + var body: some View { + VStack(alignment: .leading, spacing: 8) { + if let used = row.usedPercent { + ContextUsageBar( + usedTokens: Int(round(used)), + contextTokens: 100, + width: max(1, self.width - (self.paddingLeading + self.paddingTrailing)), + height: self.barHeight) + } + + HStack(alignment: .firstTextBaseline, spacing: 6) { + Text(self.row.titleText) + .font(.caption.weight(.semibold)) + .foregroundStyle(self.primaryTextColor) + .lineLimit(1) + .truncationMode(.middle) + .layoutPriority(1) + + Spacer(minLength: 4) + + Text(self.row.detailText()) + .font(.caption.monospacedDigit()) + .foregroundStyle(self.secondaryTextColor) + .lineLimit(1) + .truncationMode(.tail) + .layoutPriority(2) + + if self.showsChevron { + Image(systemName: "chevron.right") + .font(.caption.weight(.semibold)) + .foregroundStyle(self.secondaryTextColor) + .padding(.leading, 2) + } + } + } + .padding(.vertical, 10) + .padding(.leading, self.paddingLeading) + .padding(.trailing, self.paddingTrailing) + } +} diff --git a/apps/macos/Sources/OpenClaw/UserDefaultsMigration.swift b/apps/macos/Sources/OpenClaw/UserDefaultsMigration.swift new file mode 100644 index 0000000000000000000000000000000000000000..793e52baeb7986aa09a96544ad5b973542e88f07 --- /dev/null +++ b/apps/macos/Sources/OpenClaw/UserDefaultsMigration.swift @@ -0,0 +1,16 @@ +import Foundation + +private let legacyDefaultsPrefix = "openclaw." +private let defaultsPrefix = "openclaw." + +func migrateLegacyDefaults() { + let defaults = UserDefaults.standard + let snapshot = defaults.dictionaryRepresentation() + for (key, value) in snapshot where key.hasPrefix(legacyDefaultsPrefix) { + let suffix = key.dropFirst(legacyDefaultsPrefix.count) + let newKey = defaultsPrefix + suffix + if defaults.object(forKey: newKey) == nil { + defaults.set(value, forKey: newKey) + } + } +} diff --git a/apps/macos/Sources/OpenClaw/ViewMetrics.swift b/apps/macos/Sources/OpenClaw/ViewMetrics.swift new file mode 100644 index 0000000000000000000000000000000000000000..dfd7180de0f8187e70be42187bb525fb4bb9446b --- /dev/null +++ b/apps/macos/Sources/OpenClaw/ViewMetrics.swift @@ -0,0 +1,29 @@ +import SwiftUI + +private struct ViewWidthPreferenceKey: PreferenceKey { + static let defaultValue: CGFloat = 0 + + static func reduce(value: inout CGFloat, nextValue: () -> CGFloat) { + value = max(value, nextValue()) + } +} + +extension View { + func onWidthChange(_ onChange: @escaping (CGFloat) -> Void) -> some View { + self.background( + GeometryReader { proxy in + Color.clear.preference(key: ViewWidthPreferenceKey.self, value: proxy.size.width) + }) + .onPreferenceChange(ViewWidthPreferenceKey.self, perform: onChange) + } +} + +#if DEBUG +enum ViewMetricsTesting { + static func reduceWidth(current: CGFloat, next: CGFloat) -> CGFloat { + var value = current + ViewWidthPreferenceKey.reduce(value: &value, nextValue: { next }) + return value + } +} +#endif diff --git a/apps/macos/Sources/OpenClaw/VisualEffectView.swift b/apps/macos/Sources/OpenClaw/VisualEffectView.swift new file mode 100644 index 0000000000000000000000000000000000000000..b18971109ab56da2405859b3293ad14f0b90bbf7 --- /dev/null +++ b/apps/macos/Sources/OpenClaw/VisualEffectView.swift @@ -0,0 +1,37 @@ +import AppKit +import SwiftUI + +struct VisualEffectView: NSViewRepresentable { + var material: NSVisualEffectView.Material + var blendingMode: NSVisualEffectView.BlendingMode + var state: NSVisualEffectView.State + var emphasized: Bool + + init( + material: NSVisualEffectView.Material, + blendingMode: NSVisualEffectView.BlendingMode = .behindWindow, + state: NSVisualEffectView.State = .active, + emphasized: Bool = false) + { + self.material = material + self.blendingMode = blendingMode + self.state = state + self.emphasized = emphasized + } + + func makeNSView(context _: Context) -> NSVisualEffectView { + let view = NSVisualEffectView() + view.material = self.material + view.blendingMode = self.blendingMode + view.state = self.state + view.isEmphasized = self.emphasized + return view + } + + func updateNSView(_ nsView: NSVisualEffectView, context _: Context) { + nsView.material = self.material + nsView.blendingMode = self.blendingMode + nsView.state = self.state + nsView.isEmphasized = self.emphasized + } +} diff --git a/apps/macos/Sources/OpenClaw/VoicePushToTalk.swift b/apps/macos/Sources/OpenClaw/VoicePushToTalk.swift new file mode 100644 index 0000000000000000000000000000000000000000..819bafd1271495b825b6030f809017c66d2767cc --- /dev/null +++ b/apps/macos/Sources/OpenClaw/VoicePushToTalk.swift @@ -0,0 +1,421 @@ +import AppKit +import AVFoundation +import Dispatch +import OSLog +import Speech + +/// Observes right Option and starts a push-to-talk capture while it is held. +final class VoicePushToTalkHotkey: @unchecked Sendable { + static let shared = VoicePushToTalkHotkey() + + private var globalMonitor: Any? + private var localMonitor: Any? + private var optionDown = false // right option only + private var active = false + + private let beginAction: @Sendable () async -> Void + private let endAction: @Sendable () async -> Void + + init( + beginAction: @escaping @Sendable () async -> Void = { await VoicePushToTalk.shared.begin() }, + endAction: @escaping @Sendable () async -> Void = { await VoicePushToTalk.shared.end() }) + { + self.beginAction = beginAction + self.endAction = endAction + } + + func setEnabled(_ enabled: Bool) { + if ProcessInfo.processInfo.isRunningTests { return } + self.withMainThread { [weak self] in + guard let self else { return } + if enabled { + self.startMonitoring() + } else { + self.stopMonitoring() + } + } + } + + private func startMonitoring() { + // assert(Thread.isMainThread) - Removed for Swift 6 + guard self.globalMonitor == nil, self.localMonitor == nil else { return } + // Listen-only global monitor; we rely on Input Monitoring permission to receive events. + self.globalMonitor = NSEvent.addGlobalMonitorForEvents(matching: .flagsChanged) { [weak self] event in + let keyCode = event.keyCode + let flags = event.modifierFlags + self?.handleFlagsChanged(keyCode: keyCode, modifierFlags: flags) + } + // Also listen locally so we still catch events when the app is active/focused. + self.localMonitor = NSEvent.addLocalMonitorForEvents(matching: .flagsChanged) { [weak self] event in + let keyCode = event.keyCode + let flags = event.modifierFlags + self?.handleFlagsChanged(keyCode: keyCode, modifierFlags: flags) + return event + } + } + + private func stopMonitoring() { + // assert(Thread.isMainThread) - Removed for Swift 6 + if let globalMonitor { + NSEvent.removeMonitor(globalMonitor) + self.globalMonitor = nil + } + if let localMonitor { + NSEvent.removeMonitor(localMonitor) + self.localMonitor = nil + } + self.optionDown = false + self.active = false + } + + private func handleFlagsChanged(keyCode: UInt16, modifierFlags: NSEvent.ModifierFlags) { + self.withMainThread { [weak self] in + self?.updateModifierState(keyCode: keyCode, modifierFlags: modifierFlags) + } + } + + private func withMainThread(_ block: @escaping @Sendable () -> Void) { + DispatchQueue.main.async(execute: block) + } + + private func updateModifierState(keyCode: UInt16, modifierFlags: NSEvent.ModifierFlags) { + // assert(Thread.isMainThread) - Removed for Swift 6 + // Right Option (keyCode 61) acts as a hold-to-talk modifier. + if keyCode == 61 { + self.optionDown = modifierFlags.contains(.option) + } + + let chordActive = self.optionDown + if chordActive, !self.active { + self.active = true + Task { + Logger(subsystem: "ai.openclaw", category: "voicewake.ptt") + .info("ptt hotkey down") + await self.beginAction() + } + } else if !chordActive, self.active { + self.active = false + Task { + Logger(subsystem: "ai.openclaw", category: "voicewake.ptt") + .info("ptt hotkey up") + await self.endAction() + } + } + } + + func _testUpdateModifierState(keyCode: UInt16, modifierFlags: NSEvent.ModifierFlags) { + self.updateModifierState(keyCode: keyCode, modifierFlags: modifierFlags) + } +} + +/// Short-lived speech recognizer that records while the hotkey is held. +actor VoicePushToTalk { + static let shared = VoicePushToTalk() + + private let logger = Logger(subsystem: "ai.openclaw", category: "voicewake.ptt") + + private var recognizer: SFSpeechRecognizer? + // Lazily created on begin() to avoid creating an AVAudioEngine at app launch, which can switch Bluetooth + // headphones into the low-quality headset profile even if push-to-talk is never used. + private var audioEngine: AVAudioEngine? + private var recognitionRequest: SFSpeechAudioBufferRecognitionRequest? + private var recognitionTask: SFSpeechRecognitionTask? + private var tapInstalled = false + + // Session token used to drop stale callbacks when a new capture starts. + private var sessionID = UUID() + + private var committed: String = "" + private var volatile: String = "" + private var activeConfig: Config? + private var isCapturing = false + private var triggerChimePlayed = false + private var finalized = false + private var timeoutTask: Task? + private var overlayToken: UUID? + private var adoptedPrefix: String = "" + + private struct Config { + let micID: String? + let localeID: String? + let triggerChime: VoiceWakeChime + let sendChime: VoiceWakeChime + } + + func begin() async { + guard voiceWakeSupported else { return } + guard !self.isCapturing else { return } + + // Start a fresh session and invalidate any in-flight callbacks tied to an older one. + let sessionID = UUID() + self.sessionID = sessionID + + // Ensure permissions up front. + let granted = await PermissionManager.ensureVoiceWakePermissions(interactive: true) + guard granted else { return } + + let config = await MainActor.run { self.makeConfig() } + self.activeConfig = config + self.isCapturing = true + self.triggerChimePlayed = false + self.finalized = false + self.timeoutTask?.cancel(); self.timeoutTask = nil + let snapshot = await MainActor.run { VoiceSessionCoordinator.shared.snapshot() } + self.adoptedPrefix = snapshot.visible ? snapshot.text.trimmingCharacters(in: .whitespacesAndNewlines) : "" + self.logger.info("ptt begin adopted_prefix_len=\(self.adoptedPrefix.count, privacy: .public)") + if config.triggerChime != .none { + self.triggerChimePlayed = true + await MainActor.run { VoiceWakeChimePlayer.play(config.triggerChime, reason: "ptt.trigger") } + } + // Pause the always-on wake word recognizer so both pipelines don't fight over the mic tap. + await VoiceWakeRuntime.shared.pauseForPushToTalk() + let adoptedPrefix = self.adoptedPrefix + let adoptedAttributed: NSAttributedString? = adoptedPrefix.isEmpty ? nil : Self.makeAttributed( + committed: adoptedPrefix, + volatile: "", + isFinal: false) + self.overlayToken = await MainActor.run { + VoiceSessionCoordinator.shared.startSession( + source: .pushToTalk, + text: adoptedPrefix, + attributed: adoptedAttributed, + forwardEnabled: true) + } + + do { + try await self.startRecognition(localeID: config.localeID, sessionID: sessionID) + } catch { + await MainActor.run { + VoiceWakeOverlayController.shared.dismiss() + } + self.isCapturing = false + // If push-to-talk fails to start after pausing wake-word, ensure we resume listening. + await VoiceWakeRuntime.shared.applyPushToTalkCooldown() + await VoiceWakeRuntime.shared.refresh(state: AppStateStore.shared) + } + } + + func end() async { + guard self.isCapturing else { return } + self.isCapturing = false + let sessionID = self.sessionID + + // Stop feeding Speech buffers first, then end the request. Stopping the engine here can race with + // Speech draining its converter chain (and we already stop/cancel in finalize). + if self.tapInstalled { + self.audioEngine?.inputNode.removeTap(onBus: 0) + self.tapInstalled = false + } + self.recognitionRequest?.endAudio() + + // If we captured nothing, dismiss immediately when the user lets go. + if self.committed.isEmpty, self.volatile.isEmpty, self.adoptedPrefix.isEmpty { + await self.finalize(transcriptOverride: "", reason: "emptyOnRelease", sessionID: sessionID) + return + } + + // Otherwise, give Speech a brief window to deliver the final result; then fall back. + self.timeoutTask?.cancel() + self.timeoutTask = Task { [weak self] in + try? await Task.sleep(nanoseconds: 1_500_000_000) // 1.5s grace period to await final result + await self?.finalize(transcriptOverride: nil, reason: "timeout", sessionID: sessionID) + } + } + + // MARK: - Private + + private func startRecognition(localeID: String?, sessionID: UUID) async throws { + let locale = localeID.flatMap { Locale(identifier: $0) } ?? Locale(identifier: Locale.current.identifier) + self.recognizer = SFSpeechRecognizer(locale: locale) + guard let recognizer, recognizer.isAvailable else { + throw NSError( + domain: "VoicePushToTalk", + code: 1, + userInfo: [NSLocalizedDescriptionKey: "Recognizer unavailable"]) + } + + self.recognitionRequest = SFSpeechAudioBufferRecognitionRequest() + self.recognitionRequest?.shouldReportPartialResults = true + guard let request = self.recognitionRequest else { return } + + // Lazily create the engine here so app launch doesn't grab audio resources / trigger Bluetooth HFP. + if self.audioEngine == nil { + self.audioEngine = AVAudioEngine() + } + guard let audioEngine = self.audioEngine else { return } + + let input = audioEngine.inputNode + let format = input.outputFormat(forBus: 0) + if self.tapInstalled { + input.removeTap(onBus: 0) + self.tapInstalled = false + } + // Pipe raw mic buffers into the Speech request while the chord is held. + input.installTap(onBus: 0, bufferSize: 2048, format: format) { [weak request] buffer, _ in + request?.append(buffer) + } + self.tapInstalled = true + + audioEngine.prepare() + try audioEngine.start() + + self.recognitionTask = recognizer.recognitionTask(with: request) { [weak self] result, error in + guard let self else { return } + if let error { + self.logger.debug("push-to-talk error: \(error.localizedDescription, privacy: .public)") + } + let transcript = result?.bestTranscription.formattedString + let isFinal = result?.isFinal ?? false + // Hop to a Task so UI updates stay off the Speech callback thread. + Task.detached { [weak self, transcript, isFinal, sessionID] in + guard let self else { return } + await self.handle(transcript: transcript, isFinal: isFinal, sessionID: sessionID) + } + } + } + + private func handle(transcript: String?, isFinal: Bool, sessionID: UUID) async { + guard sessionID == self.sessionID else { + self.logger.debug("push-to-talk drop transcript for stale session") + return + } + guard let transcript else { return } + if isFinal { + self.committed = transcript + self.volatile = "" + } else { + self.volatile = Self.delta(after: self.committed, current: transcript) + } + + let committedWithPrefix = Self.join(self.adoptedPrefix, self.committed) + let snapshot = Self.join(committedWithPrefix, self.volatile) + let attributed = Self.makeAttributed(committed: committedWithPrefix, volatile: self.volatile, isFinal: isFinal) + if let token = self.overlayToken { + await MainActor.run { + VoiceSessionCoordinator.shared.updatePartial( + token: token, + text: snapshot, + attributed: attributed) + } + } + } + + private func finalize(transcriptOverride: String?, reason: String, sessionID: UUID?) async { + if self.finalized { return } + if let sessionID, sessionID != self.sessionID { + self.logger.debug("push-to-talk drop finalize for stale session") + return + } + self.finalized = true + self.isCapturing = false + self.timeoutTask?.cancel(); self.timeoutTask = nil + + let finalRecognized: String = { + if let override = transcriptOverride?.trimmingCharacters(in: .whitespacesAndNewlines) { + return override + } + return (self.committed + self.volatile).trimmingCharacters(in: .whitespacesAndNewlines) + }() + let finalText = Self.join(self.adoptedPrefix, finalRecognized) + let chime = finalText.isEmpty ? .none : (self.activeConfig?.sendChime ?? .none) + + let token = self.overlayToken + let logger = self.logger + await MainActor.run { + logger.info("ptt finalize reason=\(reason, privacy: .public) len=\(finalText.count, privacy: .public)") + if let token { + VoiceSessionCoordinator.shared.finalize( + token: token, + text: finalText, + sendChime: chime, + autoSendAfter: nil) + VoiceSessionCoordinator.shared.sendNow(token: token, reason: reason) + } else if !finalText.isEmpty { + if chime != .none { + VoiceWakeChimePlayer.play(chime, reason: "ptt.fallback_send") + } + Task.detached { + await VoiceWakeForwarder.forward(transcript: finalText) + } + } + } + + self.recognitionTask?.cancel() + self.recognitionRequest = nil + self.recognitionTask = nil + if self.tapInstalled { + self.audioEngine?.inputNode.removeTap(onBus: 0) + self.tapInstalled = false + } + if self.audioEngine?.isRunning == true { + self.audioEngine?.stop() + self.audioEngine?.reset() + } + // Release the engine so we also release any audio session/resources when push-to-talk ends. + self.audioEngine = nil + + self.committed = "" + self.volatile = "" + self.activeConfig = nil + self.triggerChimePlayed = false + self.overlayToken = nil + self.adoptedPrefix = "" + + // Resume the wake-word runtime after push-to-talk finishes. + await VoiceWakeRuntime.shared.applyPushToTalkCooldown() + _ = await MainActor.run { Task { await VoiceWakeRuntime.shared.refresh(state: AppStateStore.shared) } } + } + + @MainActor + private func makeConfig() -> Config { + let state = AppStateStore.shared + return Config( + micID: state.voiceWakeMicID.isEmpty ? nil : state.voiceWakeMicID, + localeID: state.voiceWakeLocaleID, + triggerChime: state.voiceWakeTriggerChime, + sendChime: state.voiceWakeSendChime) + } + + // MARK: - Test helpers + + static func _testDelta(committed: String, current: String) -> String { + self.delta(after: committed, current: current) + } + + static func _testAttributedColors(isFinal: Bool) -> (NSColor, NSColor) { + let sample = self.makeAttributed(committed: "a", volatile: "b", isFinal: isFinal) + let committedColor = sample.attribute(.foregroundColor, at: 0, effectiveRange: nil) as? NSColor ?? .clear + let volatileColor = sample.attribute(.foregroundColor, at: 1, effectiveRange: nil) as? NSColor ?? .clear + return (committedColor, volatileColor) + } + + private static func join(_ prefix: String, _ suffix: String) -> String { + if prefix.isEmpty { return suffix } + if suffix.isEmpty { return prefix } + return "\(prefix) \(suffix)" + } + + private static func delta(after committed: String, current: String) -> String { + if current.hasPrefix(committed) { + let start = current.index(current.startIndex, offsetBy: committed.count) + return String(current[start...]) + } + return current + } + + private static func makeAttributed(committed: String, volatile: String, isFinal: Bool) -> NSAttributedString { + let full = NSMutableAttributedString() + let committedAttr: [NSAttributedString.Key: Any] = [ + .foregroundColor: NSColor.labelColor, + .font: NSFont.systemFont(ofSize: 13, weight: .regular), + ] + full.append(NSAttributedString(string: committed, attributes: committedAttr)) + let volatileColor: NSColor = isFinal ? .labelColor : NSColor.tertiaryLabelColor + let volatileAttr: [NSAttributedString.Key: Any] = [ + .foregroundColor: volatileColor, + .font: NSFont.systemFont(ofSize: 13, weight: .regular), + ] + full.append(NSAttributedString(string: volatile, attributes: volatileAttr)) + return full + } +} diff --git a/apps/macos/Sources/OpenClaw/VoiceSessionCoordinator.swift b/apps/macos/Sources/OpenClaw/VoiceSessionCoordinator.swift new file mode 100644 index 0000000000000000000000000000000000000000..87c32d26670e7ef79747652fdde58d991cf57412 --- /dev/null +++ b/apps/macos/Sources/OpenClaw/VoiceSessionCoordinator.swift @@ -0,0 +1,134 @@ +import AppKit +import Foundation +import Observation + +@MainActor +@Observable +final class VoiceSessionCoordinator { + static let shared = VoiceSessionCoordinator() + + enum Source: String { case wakeWord, pushToTalk } + + struct Session { + let token: UUID + let source: Source + var text: String + var attributed: NSAttributedString? + var isFinal: Bool + var sendChime: VoiceWakeChime + var autoSendDelay: TimeInterval? + } + + private let logger = Logger(subsystem: "ai.openclaw", category: "voicewake.coordinator") + private var session: Session? + + // MARK: - API + + func startSession( + source: Source, + text: String, + attributed: NSAttributedString? = nil, + forwardEnabled: Bool = false) -> UUID + { + let token = UUID() + self.logger.info("coordinator start token=\(token.uuidString) source=\(source.rawValue) len=\(text.count)") + let attributedText = attributed ?? VoiceWakeOverlayController.shared.makeAttributed(from: text) + let session = Session( + token: token, + source: source, + text: text, + attributed: attributedText, + isFinal: false, + sendChime: .none, + autoSendDelay: nil) + self.session = session + VoiceWakeOverlayController.shared.startSession( + token: token, + source: VoiceWakeOverlayController.Source(rawValue: source.rawValue) ?? .wakeWord, + transcript: text, + attributed: attributedText, + forwardEnabled: forwardEnabled, + isFinal: false) + return token + } + + func updatePartial(token: UUID, text: String, attributed: NSAttributedString? = nil) { + guard let session, session.token == token else { return } + self.session?.text = text + self.session?.attributed = attributed + VoiceWakeOverlayController.shared.updatePartial(token: token, transcript: text, attributed: attributed) + } + + func finalize( + token: UUID, + text: String, + sendChime: VoiceWakeChime, + autoSendAfter: TimeInterval?) + { + guard let session, session.token == token else { return } + self.logger + .info( + "coordinator finalize token=\(token.uuidString) len=\(text.count) autoSendAfter=\(autoSendAfter ?? -1)") + self.session?.text = text + self.session?.isFinal = true + self.session?.sendChime = sendChime + self.session?.autoSendDelay = autoSendAfter + + let attributed = VoiceWakeOverlayController.shared.makeAttributed(from: text) + VoiceWakeOverlayController.shared.presentFinal( + token: token, + transcript: text, + autoSendAfter: autoSendAfter, + sendChime: sendChime, + attributed: attributed) + } + + func sendNow(token: UUID, reason: String = "explicit") { + guard let session, session.token == token else { return } + let text = session.text.trimmingCharacters(in: .whitespacesAndNewlines) + guard !text.isEmpty else { + self.logger.info("coordinator sendNow \(reason) empty -> dismiss") + VoiceWakeOverlayController.shared.dismiss(token: token, reason: .empty, outcome: .empty) + self.clearSession() + return + } + VoiceWakeOverlayController.shared.beginSendUI(token: token, sendChime: session.sendChime) + Task.detached { + _ = await VoiceWakeForwarder.forward(transcript: text) + } + } + + func dismiss( + token: UUID, + reason: VoiceWakeOverlayController.DismissReason, + outcome: VoiceWakeOverlayController.SendOutcome) + { + guard let session, session.token == token else { return } + VoiceWakeOverlayController.shared.dismiss(token: token, reason: reason, outcome: outcome) + self.clearSession() + } + + func updateLevel(token: UUID, _ level: Double) { + guard let session, session.token == token else { return } + VoiceWakeOverlayController.shared.updateLevel(token: token, level) + } + + func snapshot() -> (token: UUID?, text: String, visible: Bool) { + (self.session?.token, self.session?.text ?? "", VoiceWakeOverlayController.shared.isVisible) + } + + // MARK: - Private + + private func clearSession() { + self.session = nil + } + + /// Overlay dismiss completion callback (manual X, empty, auto-dismiss after send). + /// Ensures the wake-word recognizer is resumed if Voice Wake is enabled. + func overlayDidDismiss(token: UUID?) { + if let token, self.session?.token == token { + self.clearSession() + } + Task { await VoiceWakeRuntime.shared.refresh(state: AppStateStore.shared) } + } +} diff --git a/apps/macos/Sources/OpenClaw/VoiceWakeChime.swift b/apps/macos/Sources/OpenClaw/VoiceWakeChime.swift new file mode 100644 index 0000000000000000000000000000000000000000..c41ecf4fd435841a6f5666cec3028cfd5e6b40c3 --- /dev/null +++ b/apps/macos/Sources/OpenClaw/VoiceWakeChime.swift @@ -0,0 +1,74 @@ +import AppKit +import Foundation +import OSLog + +enum VoiceWakeChime: Codable, Equatable, Sendable { + case none + case system(name: String) + case custom(displayName: String, bookmark: Data) + + var systemName: String? { + if case let .system(name) = self { + return name + } + return nil + } + + var displayLabel: String { + switch self { + case .none: + "No Sound" + case let .system(name): + VoiceWakeChimeCatalog.displayName(for: name) + case let .custom(displayName, _): + displayName + } + } +} + +enum VoiceWakeChimeCatalog { + /// Options shown in the picker. + static var systemOptions: [String] { SoundEffectCatalog.systemOptions } + + static func displayName(for raw: String) -> String { + SoundEffectCatalog.displayName(for: raw) + } + + static func url(for name: String) -> URL? { + SoundEffectCatalog.url(for: name) + } +} + +@MainActor +enum VoiceWakeChimePlayer { + private static let logger = Logger(subsystem: "ai.openclaw", category: "voicewake.chime") + private static var lastSound: NSSound? + + static func play(_ chime: VoiceWakeChime, reason: String? = nil) { + guard let sound = self.sound(for: chime) else { return } + if let reason { + self.logger.log(level: .info, "chime play reason=\(reason, privacy: .public)") + } else { + self.logger.log(level: .info, "chime play") + } + DiagnosticsFileLog.shared.log(category: "voicewake.chime", event: "play", fields: [ + "reason": reason ?? "", + "chime": chime.displayLabel, + "systemName": chime.systemName ?? "", + ]) + SoundEffectPlayer.play(sound) + } + + private static func sound(for chime: VoiceWakeChime) -> NSSound? { + switch chime { + case .none: + nil + + case let .system(name): + SoundEffectPlayer.sound(named: name) + + case let .custom(_, bookmark): + SoundEffectPlayer.sound(from: bookmark) + } + } +} diff --git a/apps/macos/Sources/OpenClaw/VoiceWakeForwarder.swift b/apps/macos/Sources/OpenClaw/VoiceWakeForwarder.swift new file mode 100644 index 0000000000000000000000000000000000000000..ee634a628ed433233638ad7b18edf2310be6b7f8 --- /dev/null +++ b/apps/macos/Sources/OpenClaw/VoiceWakeForwarder.swift @@ -0,0 +1,73 @@ +import Foundation +import OSLog + +enum VoiceWakeForwarder { + private static let logger = Logger(subsystem: "ai.openclaw", category: "voicewake.forward") + + static func prefixedTranscript(_ transcript: String, machineName: String? = nil) -> String { + let resolvedMachine = machineName + .flatMap { name -> String? in + let trimmed = name.trimmingCharacters(in: .whitespacesAndNewlines) + return trimmed.isEmpty ? nil : trimmed + } + ?? Host.current().localizedName + ?? ProcessInfo.processInfo.hostName + + let safeMachine = resolvedMachine.isEmpty ? "this Mac" : resolvedMachine + return """ + User talked via voice recognition on \(safeMachine) - repeat prompt first \ + + remember some words might be incorrectly transcribed. + + \(transcript) + """ + } + + enum VoiceWakeForwardError: LocalizedError, Equatable { + case rpcFailed(String) + + var errorDescription: String? { + switch self { + case let .rpcFailed(message): message + } + } + } + + struct ForwardOptions: Sendable { + var sessionKey: String = "main" + var thinking: String = "low" + var deliver: Bool = true + var to: String? + var channel: GatewayAgentChannel = .last + } + + @discardableResult + static func forward( + transcript: String, + options: ForwardOptions = ForwardOptions()) async -> Result + { + let payload = Self.prefixedTranscript(transcript) + let deliver = options.channel.shouldDeliver(options.deliver) + let result = await GatewayConnection.shared.sendAgent(GatewayAgentInvocation( + message: payload, + sessionKey: options.sessionKey, + thinking: options.thinking, + deliver: deliver, + to: options.to, + channel: options.channel)) + + if result.ok { + self.logger.info("voice wake forward ok") + return .success(()) + } + + let message = result.error ?? "agent rpc unavailable" + self.logger.error("voice wake forward failed: \(message, privacy: .public)") + return .failure(.rpcFailed(message)) + } + + static func checkConnection() async -> Result { + let status = await GatewayConnection.shared.status() + if status.ok { return .success(()) } + return .failure(.rpcFailed(status.error ?? "agent rpc unreachable")) + } +} diff --git a/apps/macos/Sources/OpenClaw/VoiceWakeGlobalSettingsSync.swift b/apps/macos/Sources/OpenClaw/VoiceWakeGlobalSettingsSync.swift new file mode 100644 index 0000000000000000000000000000000000000000..fd888c8aa4fdc8a51ea9a1bed4f9f70041dca7b7 --- /dev/null +++ b/apps/macos/Sources/OpenClaw/VoiceWakeGlobalSettingsSync.swift @@ -0,0 +1,66 @@ +import OpenClawKit +import Foundation +import OSLog + +@MainActor +final class VoiceWakeGlobalSettingsSync { + static let shared = VoiceWakeGlobalSettingsSync() + + private let logger = Logger(subsystem: "ai.openclaw", category: "voicewake.sync") + private var task: Task? + + private struct VoiceWakePayload: Codable, Equatable { + let triggers: [String] + } + + func start() { + guard self.task == nil else { return } + self.task = Task { [weak self] in + guard let self else { return } + while !Task.isCancelled { + do { + try await GatewayConnection.shared.refresh() + } catch { + // Not configured / not reachable yet. + } + + await self.refreshFromGateway() + + let stream = await GatewayConnection.shared.subscribe(bufferingNewest: 200) + for await push in stream { + if Task.isCancelled { return } + await self.handle(push: push) + } + + // If the stream finishes (gateway shutdown / reconnect), loop and resubscribe. + try? await Task.sleep(nanoseconds: 600_000_000) + } + } + } + + func stop() { + self.task?.cancel() + self.task = nil + } + + private func refreshFromGateway() async { + do { + let triggers = try await GatewayConnection.shared.voiceWakeGetTriggers() + AppStateStore.shared.applyGlobalVoiceWakeTriggers(triggers) + } catch { + // Best-effort only. + } + } + + func handle(push: GatewayPush) async { + guard case let .event(evt) = push else { return } + guard evt.event == "voicewake.changed" else { return } + guard let payload = evt.payload else { return } + do { + let decoded = try GatewayPayloadDecoding.decode(payload, as: VoiceWakePayload.self) + AppStateStore.shared.applyGlobalVoiceWakeTriggers(decoded.triggers) + } catch { + self.logger.error("failed to decode voicewake.changed: \(error.localizedDescription, privacy: .public)") + } + } +} diff --git a/apps/macos/Sources/OpenClaw/VoiceWakeHelpers.swift b/apps/macos/Sources/OpenClaw/VoiceWakeHelpers.swift new file mode 100644 index 0000000000000000000000000000000000000000..98cdc0cb58a5bac9607a144021f8db4df6551428 --- /dev/null +++ b/apps/macos/Sources/OpenClaw/VoiceWakeHelpers.swift @@ -0,0 +1,24 @@ +import Foundation + +func sanitizeVoiceWakeTriggers(_ words: [String]) -> [String] { + let cleaned = words + .map { $0.trimmingCharacters(in: .whitespacesAndNewlines) } + .filter { !$0.isEmpty } + .prefix(voiceWakeMaxWords) + .map { String($0.prefix(voiceWakeMaxWordLength)) } + return cleaned.isEmpty ? defaultVoiceWakeTriggers : cleaned +} + +func normalizeLocaleIdentifier(_ raw: String) -> String { + var trimmed = raw + if let at = trimmed.firstIndex(of: "@") { + trimmed = String(trimmed[..? + var autoSendTask: Task? + var autoSendToken: UUID? + var activeToken: UUID? + var activeSource: Source? + var lastLevelUpdate: TimeInterval = 0 + + let width: CGFloat = 360 + let padding: CGFloat = 10 + let buttonWidth: CGFloat = 36 + let spacing: CGFloat = 8 + let verticalPadding: CGFloat = 8 + let maxHeight: CGFloat = 400 + let minHeight: CGFloat = 48 + let closeOverflow: CGFloat = 10 + let levelUpdateInterval: TimeInterval = 1.0 / 12.0 + + enum DismissReason { case explicit, empty } + enum SendOutcome { case sent, empty } + enum GuardOutcome { case accept, dropMismatch, dropNoActive } + + init(enableUI: Bool = true) { + self.enableUI = enableUI + } +} diff --git a/apps/macos/Sources/OpenClaw/VoiceWakeOverlayController+Session.swift b/apps/macos/Sources/OpenClaw/VoiceWakeOverlayController+Session.swift new file mode 100644 index 0000000000000000000000000000000000000000..f021eac98593210990fa40ebc311f984ad5e7af8 --- /dev/null +++ b/apps/macos/Sources/OpenClaw/VoiceWakeOverlayController+Session.swift @@ -0,0 +1,281 @@ +import AppKit +import QuartzCore + +extension VoiceWakeOverlayController { + @discardableResult + func startSession( + token: UUID = UUID(), + source: Source, + transcript: String, + attributed: NSAttributedString? = nil, + forwardEnabled: Bool = false, + isFinal: Bool = false) -> UUID + { + let message = """ + overlay session_start source=\(source.rawValue) \ + len=\(transcript.count) + """ + self.logger.log(level: .info, "\(message)") + self.activeToken = token + self.activeSource = source + self.autoSendTask?.cancel(); self.autoSendTask = nil; self.autoSendToken = nil + self.model.text = transcript + self.model.isFinal = isFinal + self.model.forwardEnabled = forwardEnabled + self.model.isSending = false + self.model.isEditing = false + self.model.attributed = attributed ?? self.makeAttributed(from: transcript) + self.model.level = 0 + self.lastLevelUpdate = 0 + self.present() + self.updateWindowFrame(animate: true) + return token + } + + func snapshot() -> (token: UUID?, source: Source?, text: String, isVisible: Bool) { + (self.activeToken, self.activeSource, self.model.text, self.model.isVisible) + } + + func updatePartial(token: UUID, transcript: String, attributed: NSAttributedString? = nil) { + guard self.guardToken(token, context: "partial") else { return } + guard !self.model.isFinal else { return } + let message = """ + overlay partial token=\(token.uuidString) \ + len=\(transcript.count) + """ + self.logger.log(level: .info, "\(message)") + self.autoSendTask?.cancel(); self.autoSendTask = nil; self.autoSendToken = nil + self.model.text = transcript + self.model.isFinal = false + self.model.forwardEnabled = false + self.model.isSending = false + self.model.isEditing = false + self.model.attributed = attributed ?? self.makeAttributed(from: transcript) + self.model.level = 0 + self.present() + self.updateWindowFrame(animate: true) + } + + func presentFinal( + token: UUID, + transcript: String, + autoSendAfter delay: TimeInterval?, + sendChime: VoiceWakeChime = .none, + attributed: NSAttributedString? = nil) + { + guard self.guardToken(token, context: "final") else { return } + let message = """ + overlay presentFinal token=\(token.uuidString) \ + len=\(transcript.count) \ + autoSendAfter=\(delay ?? -1) \ + forwardEnabled=\(!transcript.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty) + """ + self.logger.log(level: .info, "\(message)") + self.autoSendTask?.cancel() + self.autoSendToken = token + self.model.text = transcript + self.model.isFinal = true + self.model.forwardEnabled = !transcript.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty + self.model.isSending = false + self.model.isEditing = false + self.model.attributed = attributed ?? self.makeAttributed(from: transcript) + self.model.level = 0 + self.present() + if let delay { + if delay <= 0 { + self.logger.log(level: .info, "overlay autoSend immediate token=\(token.uuidString)") + VoiceSessionCoordinator.shared.sendNow(token: token, reason: "autoSendImmediate") + } else { + self.scheduleAutoSend(token: token, after: delay) + } + } + } + + func userBeganEditing() { + self.autoSendTask?.cancel() + self.model.isSending = false + self.model.isEditing = true + } + + func cancelEditingAndDismiss() { + self.autoSendTask?.cancel() + self.model.isSending = false + self.model.isEditing = false + self.dismiss(reason: .explicit) + } + + func endEditing() { + self.model.isEditing = false + } + + func updateText(_ text: String) { + self.model.text = text + self.model.isSending = false + self.model.attributed = self.makeAttributed(from: text) + self.updateWindowFrame(animate: true) + } + + /// UI-only path: show sending state and dismiss; actual forwarding is handled by the coordinator. + func beginSendUI(token: UUID, sendChime: VoiceWakeChime = .none) { + guard self.guardToken(token, context: "beginSendUI") else { return } + self.autoSendTask?.cancel(); self.autoSendToken = nil + let message = """ + overlay beginSendUI token=\(token.uuidString) \ + isSending=\(self.model.isSending) \ + forwardEnabled=\(self.model.forwardEnabled) \ + textLen=\(self.model.text.count) + """ + self.logger.log(level: .info, "\(message)") + if self.model.isSending { return } + self.model.isEditing = false + + if sendChime != .none { + let message = "overlay beginSendUI playing sendChime=\(String(describing: sendChime))" + self.logger.log(level: .info, "\(message)") + VoiceWakeChimePlayer.play(sendChime, reason: "overlay.send") + } + + self.model.isSending = true + DispatchQueue.main.asyncAfter(deadline: .now() + 0.28) { + self.logger.log( + level: .info, + "overlay beginSendUI dismiss ticking token=\(self.activeToken?.uuidString ?? "nil")") + self.dismiss(token: token, reason: .explicit, outcome: .sent) + } + } + + func requestSend(token: UUID? = nil, reason: String = "overlay_request") { + guard self.guardToken(token, context: "requestSend") else { return } + guard let active = token ?? self.activeToken else { return } + VoiceSessionCoordinator.shared.sendNow(token: active, reason: reason) + } + + func dismiss(token: UUID? = nil, reason: DismissReason = .explicit, outcome: SendOutcome = .empty) { + guard self.guardToken(token, context: "dismiss") else { return } + let message = """ + overlay dismiss token=\(self.activeToken?.uuidString ?? "nil") \ + reason=\(String(describing: reason)) \ + outcome=\(String(describing: outcome)) \ + visible=\(self.model.isVisible) \ + sending=\(self.model.isSending) + """ + self.logger.log(level: .info, "\(message)") + self.autoSendTask?.cancel(); self.autoSendToken = nil + self.model.isSending = false + self.model.isEditing = false + + if !self.enableUI { + self.model.isVisible = false + self.model.level = 0 + self.lastLevelUpdate = 0 + self.activeToken = nil + self.activeSource = nil + return + } + guard let window else { + if ProcessInfo.processInfo.isRunningTests { + self.model.isVisible = false + self.model.level = 0 + self.activeToken = nil + self.activeSource = nil + } + return + } + let target = self.dismissTargetFrame(for: window.frame, reason: reason, outcome: outcome) + NSAnimationContext.runAnimationGroup { context in + context.duration = 0.18 + context.timingFunction = CAMediaTimingFunction(name: .easeOut) + if let target { + window.animator().setFrame(target, display: true) + } + window.animator().alphaValue = 0 + } completionHandler: { + Task { @MainActor in + let dismissedToken = self.activeToken + window.orderOut(nil) + self.model.isVisible = false + self.model.level = 0 + self.lastLevelUpdate = 0 + self.activeToken = nil + self.activeSource = nil + if outcome == .empty { + AppStateStore.shared.blinkOnce() + } else if outcome == .sent { + AppStateStore.shared.celebrateSend() + } + AppStateStore.shared.stopVoiceEars() + VoiceSessionCoordinator.shared.overlayDidDismiss(token: dismissedToken) + } + } + } + + func updateLevel(token: UUID, _ level: Double) { + guard self.guardToken(token, context: "level") else { return } + guard self.model.isVisible else { return } + let now = ProcessInfo.processInfo.systemUptime + if level != 0, now - self.lastLevelUpdate < self.levelUpdateInterval { + return + } + self.lastLevelUpdate = now + self.model.level = max(0, min(1, level)) + } + + private func guardToken(_ token: UUID?, context: String) -> Bool { + switch Self.evaluateToken(active: self.activeToken, incoming: token) { + case .accept: + return true + case .dropMismatch: + self.logger.log( + level: .info, + """ + overlay drop \(context, privacy: .public) token_mismatch \ + active=\(self.activeToken?.uuidString ?? "nil", privacy: .public) \ + got=\(token?.uuidString ?? "nil", privacy: .public) + """) + return false + case .dropNoActive: + self.logger.log(level: .info, "overlay drop \(context, privacy: .public) no_active") + return false + } + } + + nonisolated static func evaluateToken(active: UUID?, incoming: UUID?) -> GuardOutcome { + guard let active else { return .dropNoActive } + if let incoming, incoming != active { return .dropMismatch } + return .accept + } + + func scheduleAutoSend(token: UUID, after delay: TimeInterval) { + self.logger.log( + level: .info, + """ + overlay scheduleAutoSend token=\(token.uuidString) \ + after=\(delay) + """) + self.autoSendTask?.cancel() + self.autoSendToken = token + self.autoSendTask = Task { [weak self, token] in + let nanos = UInt64(max(0, delay) * 1_000_000_000) + try? await Task.sleep(nanoseconds: nanos) + guard !Task.isCancelled else { return } + await MainActor.run { + guard let self else { return } + guard self.guardToken(token, context: "autoSend") else { return } + self.logger.log( + level: .info, + "overlay autoSend firing token=\(token.uuidString, privacy: .public)") + VoiceSessionCoordinator.shared.sendNow(token: token, reason: "autoSendDelay") + self.autoSendTask = nil + } + } + } + + func makeAttributed(from text: String) -> NSAttributedString { + NSAttributedString( + string: text, + attributes: [ + .foregroundColor: NSColor.labelColor, + .font: NSFont.systemFont(ofSize: 13, weight: .regular), + ]) + } +} diff --git a/apps/macos/Sources/OpenClaw/VoiceWakeOverlayController+Testing.swift b/apps/macos/Sources/OpenClaw/VoiceWakeOverlayController+Testing.swift new file mode 100644 index 0000000000000000000000000000000000000000..af1111df909ac75ea3b9d1f4ceaaf0e001d85521 --- /dev/null +++ b/apps/macos/Sources/OpenClaw/VoiceWakeOverlayController+Testing.swift @@ -0,0 +1,49 @@ +import AppKit + +#if DEBUG +@MainActor +extension VoiceWakeOverlayController { + static func exerciseForTesting() async { + let controller = VoiceWakeOverlayController(enableUI: false) + let token = controller.startSession( + source: .wakeWord, + transcript: "Hello", + attributed: nil, + forwardEnabled: true, + isFinal: false) + + controller.updatePartial(token: token, transcript: "Hello world") + controller.presentFinal(token: token, transcript: "Final", autoSendAfter: nil) + controller.userBeganEditing() + controller.endEditing() + controller.updateText("Edited text") + + _ = controller.makeAttributed(from: "Attributed") + _ = controller.targetFrame() + _ = controller.measuredHeight() + _ = controller.dismissTargetFrame( + for: NSRect(x: 0, y: 0, width: 120, height: 60), + reason: .empty, + outcome: .empty) + _ = controller.dismissTargetFrame( + for: NSRect(x: 0, y: 0, width: 120, height: 60), + reason: .explicit, + outcome: .sent) + _ = controller.dismissTargetFrame( + for: NSRect(x: 0, y: 0, width: 120, height: 60), + reason: .explicit, + outcome: .empty) + + controller.beginSendUI(token: token, sendChime: .none) + try? await Task.sleep(nanoseconds: 350_000_000) + + controller.scheduleAutoSend(token: token, after: 10) + controller.autoSendTask?.cancel() + controller.autoSendTask = nil + controller.autoSendToken = nil + + controller.dismiss(token: token, reason: .explicit, outcome: .sent) + controller.bringToFrontIfVisible() + } +} +#endif diff --git a/apps/macos/Sources/OpenClaw/VoiceWakeOverlayController+Window.swift b/apps/macos/Sources/OpenClaw/VoiceWakeOverlayController+Window.swift new file mode 100644 index 0000000000000000000000000000000000000000..fb5526a8d450e33a5074df0c3d63d9d13d58038b --- /dev/null +++ b/apps/macos/Sources/OpenClaw/VoiceWakeOverlayController+Window.swift @@ -0,0 +1,141 @@ +import AppKit +import QuartzCore +import SwiftUI + +extension VoiceWakeOverlayController { + func present() { + if !self.enableUI || ProcessInfo.processInfo.isRunningTests { + if !self.model.isVisible { + self.model.isVisible = true + } + return + } + self.ensureWindow() + self.hostingView?.rootView = VoiceWakeOverlayView(controller: self) + let target = self.targetFrame() + + guard let window else { return } + if !self.model.isVisible { + self.model.isVisible = true + self.logger.log( + level: .info, + "overlay present windowShown textLen=\(self.model.text.count, privacy: .public)") + // Keep the status item in “listening” mode until we explicitly dismiss the overlay. + AppStateStore.shared.triggerVoiceEars(ttl: nil) + let start = target.offsetBy(dx: 0, dy: -6) + window.setFrame(start, display: true) + window.alphaValue = 0 + window.orderFrontRegardless() + NSAnimationContext.runAnimationGroup { context in + context.duration = 0.18 + context.timingFunction = CAMediaTimingFunction(name: .easeOut) + window.animator().setFrame(target, display: true) + window.animator().alphaValue = 1 + } + } else { + self.updateWindowFrame(animate: true) + window.orderFrontRegardless() + } + } + + private func ensureWindow() { + if self.window != nil { return } + let borderPad = self.closeOverflow + let panel = NSPanel( + contentRect: NSRect(x: 0, y: 0, width: self.width + borderPad * 2, height: 60 + borderPad * 2), + styleMask: [.nonactivatingPanel, .borderless], + backing: .buffered, + defer: false) + panel.isOpaque = false + panel.backgroundColor = .clear + panel.hasShadow = false + panel.level = Self.preferredWindowLevel + panel.collectionBehavior = [.canJoinAllSpaces, .fullScreenAuxiliary, .transient] + panel.hidesOnDeactivate = false + panel.isMovable = false + panel.isFloatingPanel = true + panel.becomesKeyOnlyIfNeeded = true + panel.titleVisibility = .hidden + panel.titlebarAppearsTransparent = true + + let host = NSHostingView(rootView: VoiceWakeOverlayView(controller: self)) + host.translatesAutoresizingMaskIntoConstraints = false + panel.contentView = host + self.hostingView = host + self.window = panel + } + + /// Reassert window ordering when other panels are shown. + func bringToFrontIfVisible() { + guard self.model.isVisible, let window = self.window else { return } + window.level = Self.preferredWindowLevel + window.orderFrontRegardless() + } + + func targetFrame() -> NSRect { + guard let screen = NSScreen.main else { return .zero } + let height = self.measuredHeight() + let size = NSSize(width: self.width + self.closeOverflow * 2, height: height + self.closeOverflow * 2) + let visible = screen.visibleFrame + let origin = CGPoint( + x: visible.maxX - size.width, + y: visible.maxY - size.height) + return NSRect(origin: origin, size: size) + } + + func updateWindowFrame(animate: Bool = false) { + guard let window else { return } + let frame = self.targetFrame() + if animate { + NSAnimationContext.runAnimationGroup { context in + context.duration = 0.12 + context.timingFunction = CAMediaTimingFunction(name: .easeOut) + window.animator().setFrame(frame, display: true) + } + } else { + window.setFrame(frame, display: true) + } + } + + func measuredHeight() -> CGFloat { + let attributed = self.model.attributed.length > 0 ? self.model.attributed : self + .makeAttributed(from: self.model.text) + let maxWidth = self.width - (self.padding * 2) - self.spacing - self.buttonWidth + + let textInset = NSSize(width: 2, height: 6) + let lineFragmentPadding: CGFloat = 0 + let containerWidth = max(1, maxWidth - (textInset.width * 2) - (lineFragmentPadding * 2)) + + let storage = NSTextStorage(attributedString: attributed) + let container = NSTextContainer(containerSize: CGSize(width: containerWidth, height: .greatestFiniteMagnitude)) + container.lineFragmentPadding = lineFragmentPadding + container.lineBreakMode = .byWordWrapping + + let layout = NSLayoutManager() + layout.addTextContainer(container) + storage.addLayoutManager(layout) + + _ = layout.glyphRange(for: container) + let used = layout.usedRect(for: container) + + let contentHeight = ceil(used.height + (textInset.height * 2)) + let total = contentHeight + self.verticalPadding * 2 + self.model.isOverflowing = total > self.maxHeight + return max(self.minHeight, min(total, self.maxHeight)) + } + + func dismissTargetFrame(for frame: NSRect, reason: DismissReason, outcome: SendOutcome) -> NSRect? { + switch (reason, outcome) { + case (.empty, _): + let scale: CGFloat = 0.95 + let newSize = NSSize(width: frame.size.width * scale, height: frame.size.height * scale) + let dx = (frame.size.width - newSize.width) / 2 + let dy = (frame.size.height - newSize.height) / 2 + return NSRect(x: frame.origin.x + dx, y: frame.origin.y + dy, width: newSize.width, height: newSize.height) + case (.explicit, .sent): + return frame.offsetBy(dx: 8, dy: 6) + default: + return frame + } + } +} diff --git a/apps/macos/Sources/OpenClaw/VoiceWakeOverlayTextViews.swift b/apps/macos/Sources/OpenClaw/VoiceWakeOverlayTextViews.swift new file mode 100644 index 0000000000000000000000000000000000000000..151db8c9324d509fd61890a6bd93f3978962563c --- /dev/null +++ b/apps/macos/Sources/OpenClaw/VoiceWakeOverlayTextViews.swift @@ -0,0 +1,196 @@ +import AppKit +import SwiftUI + +struct TranscriptTextView: NSViewRepresentable { + @Binding var text: String + var attributed: NSAttributedString + var isFinal: Bool + var isOverflowing: Bool + var onBeginEditing: () -> Void + var onEscape: () -> Void + var onEndEditing: () -> Void + var onSend: () -> Void + + func makeCoordinator() -> Coordinator { Coordinator(self) } + + func makeNSView(context: Context) -> NSScrollView { + let textView = TranscriptNSTextView() + textView.delegate = context.coordinator + textView.drawsBackground = false + textView.isRichText = true + textView.isAutomaticQuoteSubstitutionEnabled = false + textView.isAutomaticTextReplacementEnabled = false + textView.font = .systemFont(ofSize: 13, weight: .regular) + textView.textContainer?.lineBreakMode = .byWordWrapping + textView.textContainer?.lineFragmentPadding = 0 + textView.textContainerInset = NSSize(width: 2, height: 6) + + textView.minSize = .zero + textView.maxSize = NSSize(width: CGFloat.greatestFiniteMagnitude, height: CGFloat.greatestFiniteMagnitude) + textView.isHorizontallyResizable = false + textView.isVerticallyResizable = true + textView.autoresizingMask = [.width] + + textView.textContainer?.containerSize = NSSize(width: 0, height: CGFloat.greatestFiniteMagnitude) + textView.textContainer?.widthTracksTextView = true + + textView.textStorage?.setAttributedString(self.attributed) + textView.typingAttributes = [ + .foregroundColor: NSColor.labelColor, + .font: NSFont.systemFont(ofSize: 13, weight: .regular), + ] + textView.focusRingType = .none + textView.onSend = { [weak textView] in + textView?.window?.makeFirstResponder(nil) + self.onSend() + } + textView.onBeginEditing = self.onBeginEditing + textView.onEscape = self.onEscape + textView.onEndEditing = self.onEndEditing + + let scroll = NSScrollView() + scroll.drawsBackground = false + scroll.borderType = .noBorder + scroll.hasVerticalScroller = true + scroll.autohidesScrollers = true + scroll.scrollerStyle = .overlay + scroll.hasHorizontalScroller = false + scroll.documentView = textView + return scroll + } + + func updateNSView(_ scrollView: NSScrollView, context: Context) { + guard let textView = scrollView.documentView as? TranscriptNSTextView else { return } + let isEditing = scrollView.window?.firstResponder == textView + if isEditing { + return + } + + if !textView.attributedString().isEqual(to: self.attributed) { + context.coordinator.isProgrammaticUpdate = true + defer { context.coordinator.isProgrammaticUpdate = false } + textView.textStorage?.setAttributedString(self.attributed) + } + } + + final class Coordinator: NSObject, NSTextViewDelegate { + var parent: TranscriptTextView + var isProgrammaticUpdate = false + + init(_ parent: TranscriptTextView) { self.parent = parent } + + func textDidBeginEditing(_ notification: Notification) { + self.parent.onBeginEditing() + } + + func textDidEndEditing(_ notification: Notification) { + self.parent.onEndEditing() + } + + func textDidChange(_ notification: Notification) { + guard !self.isProgrammaticUpdate else { return } + guard let view = notification.object as? NSTextView else { return } + guard view.window?.firstResponder === view else { return } + self.parent.text = view.string + } + } +} + +// MARK: - Vibrant display label + +struct VibrantLabelView: NSViewRepresentable { + var attributed: NSAttributedString + var onTap: () -> Void + + func makeNSView(context: Context) -> NSView { + let label = NSTextField(labelWithAttributedString: self.attributed) + label.isEditable = false + label.isBordered = false + label.drawsBackground = false + label.lineBreakMode = .byWordWrapping + label.maximumNumberOfLines = 0 + label.usesSingleLineMode = false + label.cell?.wraps = true + label.cell?.isScrollable = false + label.setContentHuggingPriority(.defaultLow, for: .horizontal) + label.setContentCompressionResistancePriority(.defaultLow, for: .horizontal) + label.setContentHuggingPriority(.required, for: .vertical) + label.setContentCompressionResistancePriority(.required, for: .vertical) + label.textColor = .labelColor + + let container = ClickCatcher(onTap: onTap) + container.addSubview(label) + + label.translatesAutoresizingMaskIntoConstraints = false + NSLayoutConstraint.activate([ + label.leadingAnchor.constraint(equalTo: container.leadingAnchor), + label.trailingAnchor.constraint(equalTo: container.trailingAnchor), + label.topAnchor.constraint(equalTo: container.topAnchor), + label.bottomAnchor.constraint(equalTo: container.bottomAnchor), + ]) + return container + } + + func updateNSView(_ nsView: NSView, context: Context) { + guard let container = nsView as? ClickCatcher, + let label = container.subviews.first as? NSTextField else { return } + label.attributedStringValue = self.attributed.strippingForegroundColor() + label.textColor = .labelColor + } +} + +private final class ClickCatcher: NSView { + let onTap: () -> Void + init(onTap: @escaping () -> Void) { + self.onTap = onTap + super.init(frame: .zero) + } + + @available(*, unavailable) + required init?(coder: NSCoder) { fatalError("init(coder:) has not been implemented") } + + override func mouseDown(with event: NSEvent) { + super.mouseDown(with: event) + self.onTap() + } +} + +private final class TranscriptNSTextView: NSTextView { + var onSend: (() -> Void)? + var onBeginEditing: (() -> Void)? + var onEndEditing: (() -> Void)? + var onEscape: (() -> Void)? + + override func becomeFirstResponder() -> Bool { + self.onBeginEditing?() + return super.becomeFirstResponder() + } + + override func resignFirstResponder() -> Bool { + let result = super.resignFirstResponder() + self.onEndEditing?() + return result + } + + override func keyDown(with event: NSEvent) { + let isReturn = event.keyCode == 36 + let isEscape = event.keyCode == 53 + if isEscape { + self.onEscape?() + return + } + if isReturn, event.modifierFlags.contains(.command) { + self.onSend?() + return + } + if isReturn { + if event.modifierFlags.contains(.shift) { + super.insertNewline(nil) + return + } + self.onSend?() + return + } + super.keyDown(with: event) + } +} diff --git a/apps/macos/Sources/OpenClaw/VoiceWakeOverlayView.swift b/apps/macos/Sources/OpenClaw/VoiceWakeOverlayView.swift new file mode 100644 index 0000000000000000000000000000000000000000..48055c10a6c37dd29fb62ca88ebd792c250ea8cb --- /dev/null +++ b/apps/macos/Sources/OpenClaw/VoiceWakeOverlayView.swift @@ -0,0 +1,186 @@ +import SwiftUI + +struct VoiceWakeOverlayView: View { + var controller: VoiceWakeOverlayController + @FocusState private var textFocused: Bool + @State private var isHovering: Bool = false + @State private var closeHovering: Bool = false + + var body: some View { + ZStack(alignment: .topLeading) { + HStack(alignment: .top, spacing: 8) { + if self.controller.model.isEditing { + TranscriptTextView( + text: Binding( + get: { self.controller.model.text }, + set: { self.controller.updateText($0) }), + attributed: self.controller.model.attributed, + isFinal: self.controller.model.isFinal, + isOverflowing: self.controller.model.isOverflowing, + onBeginEditing: { + self.controller.userBeganEditing() + }, + onEscape: { + self.controller.cancelEditingAndDismiss() + }, + onEndEditing: { + self.controller.endEditing() + }, + onSend: { + self.controller.requestSend() + }) + .focused(self.$textFocused) + .frame(maxWidth: .infinity, minHeight: 32, maxHeight: .infinity, alignment: .topLeading) + .id("editing") + } else { + VibrantLabelView( + attributed: self.controller.model.attributed, + onTap: { + self.controller.userBeganEditing() + self.textFocused = true + }) + .frame(maxWidth: .infinity, minHeight: 32, maxHeight: .infinity, alignment: .topLeading) + .focusable(false) + .id("display") + } + + Button { + self.controller.requestSend() + } label: { + let sending = self.controller.model.isSending + let level = self.controller.model.level + ZStack { + GeometryReader { geo in + let width = geo.size.width + RoundedRectangle(cornerRadius: 8, style: .continuous) + .fill(Color.accentColor.opacity(0.12)) + RoundedRectangle(cornerRadius: 8, style: .continuous) + .fill(Color.accentColor.opacity(0.25)) + .frame(width: width * max(0, min(1, level)), alignment: .leading) + .animation(.easeOut(duration: 0.08), value: level) + } + .frame(height: 28) + + ZStack { + Image(systemName: "paperplane.fill") + .opacity(sending ? 0 : 1) + .scaleEffect(sending ? 0.5 : 1) + Image(systemName: "checkmark.circle.fill") + .foregroundStyle(.green) + .opacity(sending ? 1 : 0) + .scaleEffect(sending ? 1.05 : 0.8) + } + .imageScale(.small) + } + .clipShape(RoundedRectangle(cornerRadius: 8, style: .continuous)) + .frame(width: 32, height: 28) + .animation(.spring(response: 0.35, dampingFraction: 0.78), value: sending) + } + .buttonStyle(.plain) + .disabled(!self.controller.model.forwardEnabled || self.controller.model.isSending) + .keyboardShortcut(.return, modifiers: [.command]) + } + .padding(.vertical, 8) + .padding(.horizontal, 10) + .frame(maxWidth: .infinity, maxHeight: .infinity, alignment: .topLeading) + .background { + OverlayBackground() + .equatable() + } + .shadow(color: Color.black.opacity(0.22), radius: 14, x: 0, y: -2) + .onHover { self.isHovering = $0 } + + // Close button rendered above and outside the clipped bubble + CloseButtonOverlay( + isVisible: self.controller.model.isEditing || self.isHovering || self.closeHovering, + onHover: { self.closeHovering = $0 }, + onClose: { self.controller.cancelEditingAndDismiss() }) + } + .padding(.top, self.controller.closeOverflow) + .padding(.leading, self.controller.closeOverflow) + .padding(.trailing, self.controller.closeOverflow) + .padding(.bottom, self.controller.closeOverflow) + .onAppear { + self.updateFocusState(visible: self.controller.model.isVisible, editing: self.controller.model.isEditing) + } + .onChange(of: self.controller.model.isVisible) { _, visible in + self.updateFocusState(visible: visible, editing: self.controller.model.isEditing) + } + .onChange(of: self.controller.model.isEditing) { _, editing in + self.updateFocusState(visible: self.controller.model.isVisible, editing: editing) + } + .onChange(of: self.controller.model.attributed) { _, _ in + self.controller.updateWindowFrame(animate: true) + } + } + + private func updateFocusState(visible: Bool, editing: Bool) { + let shouldFocus = visible && editing + guard self.textFocused != shouldFocus else { return } + self.textFocused = shouldFocus + } +} + +private struct OverlayBackground: View { + var body: some View { + let shape = RoundedRectangle(cornerRadius: 12, style: .continuous) + VisualEffectView(material: .hudWindow, blendingMode: .behindWindow) + .clipShape(shape) + .overlay(shape.strokeBorder(Color.white.opacity(0.16), lineWidth: 1)) + } +} + +extension OverlayBackground: @MainActor Equatable { + static func == (lhs: Self, rhs: Self) -> Bool { true } +} + +struct CloseHoverButton: View { + var onClose: () -> Void + + var body: some View { + Button(action: self.onClose) { + Image(systemName: "xmark") + .font(.system(size: 12, weight: .bold)) + .foregroundColor(Color.white.opacity(0.85)) + .frame(width: 22, height: 22) + .background(Color.black.opacity(0.35)) + .clipShape(Circle()) + .shadow(color: Color.black.opacity(0.35), radius: 6, y: 2) + } + .buttonStyle(.plain) + .focusable(false) + .contentShape(Circle()) + .padding(6) + } +} + +struct CloseButtonOverlay: View { + var isVisible: Bool + var onHover: (Bool) -> Void + var onClose: () -> Void + + var body: some View { + Group { + if self.isVisible { + Button(action: self.onClose) { + Image(systemName: "xmark") + .font(.system(size: 12, weight: .bold)) + .foregroundColor(Color.white.opacity(0.9)) + .frame(width: 22, height: 22) + .background(Color.black.opacity(0.4)) + .clipShape(Circle()) + .shadow(color: Color.black.opacity(0.45), radius: 10, x: 0, y: 3) + .shadow(color: Color.black.opacity(0.2), radius: 2, x: 0, y: 0) + } + .buttonStyle(.plain) + .focusable(false) + .contentShape(Circle()) + .padding(6) + .onHover { self.onHover($0) } + .offset(x: -9, y: -9) + .transition(.opacity) + } + } + .allowsHitTesting(self.isVisible) + } +} diff --git a/apps/macos/Sources/OpenClaw/VoiceWakeRuntime.swift b/apps/macos/Sources/OpenClaw/VoiceWakeRuntime.swift new file mode 100644 index 0000000000000000000000000000000000000000..5035357c870cd61324c63a65dad3aadaf61e2981 --- /dev/null +++ b/apps/macos/Sources/OpenClaw/VoiceWakeRuntime.swift @@ -0,0 +1,804 @@ +import AVFoundation +import Foundation +import OSLog +import Speech +import SwabbleKit +#if canImport(AppKit) +import AppKit +#endif + +/// Background listener that keeps the voice-wake pipeline alive outside the settings test view. +actor VoiceWakeRuntime { + static let shared = VoiceWakeRuntime() + + enum ListeningState { case idle, voiceWake, pushToTalk } + + private let logger = Logger(subsystem: "ai.openclaw", category: "voicewake.runtime") + + private var recognizer: SFSpeechRecognizer? + // Lazily created on start to avoid creating an AVAudioEngine at app launch, which can switch Bluetooth + // headphones into the low-quality headset profile even if Voice Wake is disabled. + private var audioEngine: AVAudioEngine? + private var recognitionRequest: SFSpeechAudioBufferRecognitionRequest? + private var recognitionTask: SFSpeechRecognitionTask? + private var recognitionGeneration: Int = 0 // drop stale callbacks after restarts + private var lastHeard: Date? + private var noiseFloorRMS: Double = 1e-4 + private var captureStartedAt: Date? + private var captureTask: Task? + private var capturedTranscript: String = "" + private var isCapturing: Bool = false + private var heardBeyondTrigger: Bool = false + private var triggerChimePlayed: Bool = false + private var committedTranscript: String = "" + private var volatileTranscript: String = "" + private var cooldownUntil: Date? + private var currentConfig: RuntimeConfig? + private var listeningState: ListeningState = .idle + private var overlayToken: UUID? + private var activeTriggerEndTime: TimeInterval? + private var scheduledRestartTask: Task? + private var lastLoggedText: String? + private var lastLoggedAt: Date? + private var lastTapLogAt: Date? + private var lastCallbackLogAt: Date? + private var lastTranscript: String? + private var lastTranscriptAt: Date? + private var preDetectTask: Task? + private var isStarting: Bool = false + private var triggerOnlyTask: Task? + + // Tunables + // Silence threshold once we've captured user speech (post-trigger). + private let silenceWindow: TimeInterval = 2.0 + // Silence threshold when we only heard the trigger but no post-trigger speech yet. + private let triggerOnlySilenceWindow: TimeInterval = 5.0 + // Maximum capture duration from trigger until we force-send, to avoid runaway sessions. + private let captureHardStop: TimeInterval = 120.0 + private let debounceAfterSend: TimeInterval = 0.35 + // Voice activity detection parameters (RMS-based). + private let minSpeechRMS: Double = 1e-3 + private let speechBoostFactor: Double = 6.0 // how far above noise floor we require to mark speech + private let preDetectSilenceWindow: TimeInterval = 1.0 + private let triggerPauseWindow: TimeInterval = 0.55 + + /// Stops the active Speech pipeline without clearing the stored config, so we can restart cleanly. + private func haltRecognitionPipeline() { + // Bump generation first so any in-flight callbacks from the cancelled task get dropped. + self.recognitionGeneration &+= 1 + self.recognitionTask?.cancel() + self.recognitionTask = nil + self.recognitionRequest?.endAudio() + self.recognitionRequest = nil + self.audioEngine?.inputNode.removeTap(onBus: 0) + self.audioEngine?.stop() + // Release the engine so we also release any audio session/resources when Voice Wake is idle. + self.audioEngine = nil + } + + struct RuntimeConfig: Equatable { + let triggers: [String] + let micID: String? + let localeID: String? + let triggerChime: VoiceWakeChime + let sendChime: VoiceWakeChime + } + + private struct RecognitionUpdate { + let transcript: String? + let segments: [WakeWordSegment] + let isFinal: Bool + let error: Error? + let generation: Int + } + + func refresh(state: AppState) async { + let snapshot = await MainActor.run { () -> (Bool, RuntimeConfig) in + let enabled = state.swabbleEnabled + let config = RuntimeConfig( + triggers: sanitizeVoiceWakeTriggers(state.swabbleTriggerWords), + micID: state.voiceWakeMicID.isEmpty ? nil : state.voiceWakeMicID, + localeID: state.voiceWakeLocaleID.isEmpty ? nil : state.voiceWakeLocaleID, + triggerChime: state.voiceWakeTriggerChime, + sendChime: state.voiceWakeSendChime) + return (enabled, config) + } + + guard voiceWakeSupported, snapshot.0 else { + self.stop() + return + } + + guard PermissionManager.voiceWakePermissionsGranted() else { + self.logger.debug("voicewake runtime not starting: permissions missing") + self.stop() + return + } + + let config = snapshot.1 + + if self.isStarting { + return + } + + if self.scheduledRestartTask != nil, config == self.currentConfig, self.recognitionTask == nil { + return + } + + if self.scheduledRestartTask != nil { + self.scheduledRestartTask?.cancel() + self.scheduledRestartTask = nil + } + + if config == self.currentConfig, self.recognitionTask != nil { + return + } + + self.stop() + await self.start(with: config) + } + + private func start(with config: RuntimeConfig) async { + if self.isStarting { + return + } + self.isStarting = true + defer { self.isStarting = false } + do { + self.recognitionGeneration &+= 1 + let generation = self.recognitionGeneration + + self.configureSession(localeID: config.localeID) + + guard let recognizer, recognizer.isAvailable else { + self.logger.error("voicewake runtime: speech recognizer unavailable") + return + } + + self.recognitionRequest = SFSpeechAudioBufferRecognitionRequest() + self.recognitionRequest?.shouldReportPartialResults = true + self.recognitionRequest?.taskHint = .dictation + guard let request = self.recognitionRequest else { return } + + // Lazily create the engine here so app launch doesn't grab audio resources / trigger Bluetooth HFP. + if self.audioEngine == nil { + self.audioEngine = AVAudioEngine() + } + guard let audioEngine = self.audioEngine else { return } + + let input = audioEngine.inputNode + let format = input.outputFormat(forBus: 0) + guard format.channelCount > 0, format.sampleRate > 0 else { + throw NSError( + domain: "VoiceWakeRuntime", + code: 1, + userInfo: [NSLocalizedDescriptionKey: "No audio input available"]) + } + input.removeTap(onBus: 0) + input.installTap(onBus: 0, bufferSize: 2048, format: format) { [weak self, weak request] buffer, _ in + request?.append(buffer) + guard let rms = Self.rmsLevel(buffer: buffer) else { return } + Task.detached { [weak self] in + await self?.noteAudioLevel(rms: rms) + await self?.noteAudioTap(rms: rms) + } + } + + audioEngine.prepare() + try audioEngine.start() + + self.currentConfig = config + self.lastHeard = Date() + // Preserve any existing cooldownUntil so the debounce after send isn't wiped by a restart. + + self.recognitionTask = recognizer.recognitionTask(with: request) { [weak self, generation] result, error in + guard let self else { return } + let transcript = result?.bestTranscription.formattedString + let segments = result.flatMap { result in + transcript + .map { WakeWordSpeechSegments.from(transcription: result.bestTranscription, transcript: $0) } + } ?? [] + let isFinal = result?.isFinal ?? false + Task { await self.noteRecognitionCallback(transcript: transcript, isFinal: isFinal, error: error) } + let update = RecognitionUpdate( + transcript: transcript, + segments: segments, + isFinal: isFinal, + error: error, + generation: generation) + Task { await self.handleRecognition(update, config: config) } + } + + let preferred = config.micID?.isEmpty == false ? config.micID! : "system-default" + self.logger.info( + "voicewake runtime input preferred=\(preferred, privacy: .public) " + + "\(AudioInputDeviceObserver.defaultInputDeviceSummary(), privacy: .public)") + self.logger.info("voicewake runtime started") + DiagnosticsFileLog.shared.log(category: "voicewake.runtime", event: "started", fields: [ + "locale": config.localeID ?? "", + "micID": config.micID ?? "", + ]) + } catch { + self.logger.error("voicewake runtime failed to start: \(error.localizedDescription, privacy: .public)") + self.stop() + } + } + + private func stop(dismissOverlay: Bool = true, cancelScheduledRestart: Bool = true) { + if cancelScheduledRestart { + self.scheduledRestartTask?.cancel() + self.scheduledRestartTask = nil + } + self.captureTask?.cancel() + self.captureTask = nil + self.isCapturing = false + self.capturedTranscript = "" + self.captureStartedAt = nil + self.triggerChimePlayed = false + self.lastTranscript = nil + self.lastTranscriptAt = nil + self.preDetectTask?.cancel() + self.preDetectTask = nil + self.triggerOnlyTask?.cancel() + self.triggerOnlyTask = nil + self.haltRecognitionPipeline() + self.recognizer = nil + self.currentConfig = nil + self.listeningState = .idle + self.activeTriggerEndTime = nil + self.logger.debug("voicewake runtime stopped") + DiagnosticsFileLog.shared.log(category: "voicewake.runtime", event: "stopped") + + let token = self.overlayToken + self.overlayToken = nil + guard dismissOverlay else { return } + Task { @MainActor in + if let token { + VoiceSessionCoordinator.shared.dismiss(token: token, reason: .explicit, outcome: .empty) + } else { + VoiceWakeOverlayController.shared.dismiss() + } + } + } + + private func configureSession(localeID: String?) { + let locale = localeID.flatMap { Locale(identifier: $0) } ?? Locale(identifier: Locale.current.identifier) + self.recognizer = SFSpeechRecognizer(locale: locale) + self.recognizer?.defaultTaskHint = .dictation + } + + private func handleRecognition(_ update: RecognitionUpdate, config: RuntimeConfig) async { + if update.generation != self.recognitionGeneration { + return // stale callback from a superseded recognizer session + } + if let error = update.error { + self.logger.debug("voicewake recognition error: \(error.localizedDescription, privacy: .public)") + } + + guard let transcript = update.transcript else { return } + + let now = Date() + if !transcript.isEmpty { + self.lastHeard = now + if !self.isCapturing { + self.lastTranscript = transcript + self.lastTranscriptAt = now + } + if self.isCapturing { + self.maybeLogRecognition( + transcript: transcript, + segments: update.segments, + triggers: config.triggers, + isFinal: update.isFinal, + match: nil, + usedFallback: false, + capturing: true) + let trimmed = Self.commandAfterTrigger( + transcript: transcript, + segments: update.segments, + triggerEndTime: self.activeTriggerEndTime, + triggers: config.triggers) + self.capturedTranscript = trimmed + self.updateHeardBeyondTrigger(withTrimmed: trimmed) + if update.isFinal { + self.committedTranscript = trimmed + self.volatileTranscript = "" + } else { + self.volatileTranscript = Self.delta(after: self.committedTranscript, current: trimmed) + } + + let attributed = Self.makeAttributed( + committed: self.committedTranscript, + volatile: self.volatileTranscript, + isFinal: update.isFinal) + let snapshot = self.committedTranscript + self.volatileTranscript + if let token = self.overlayToken { + await MainActor.run { + VoiceSessionCoordinator.shared.updatePartial( + token: token, + text: snapshot, + attributed: attributed) + } + } + } + } + + if self.isCapturing { return } + + let gateConfig = WakeWordGateConfig(triggers: config.triggers) + var usedFallback = false + var match = WakeWordGate.match(transcript: transcript, segments: update.segments, config: gateConfig) + if match == nil, update.isFinal { + match = self.textOnlyFallbackMatch( + transcript: transcript, + triggers: config.triggers, + config: gateConfig) + usedFallback = match != nil + } + self.maybeLogRecognition( + transcript: transcript, + segments: update.segments, + triggers: config.triggers, + isFinal: update.isFinal, + match: match, + usedFallback: usedFallback, + capturing: false) + + if let match { + if let cooldown = cooldownUntil, now < cooldown { + return + } + if usedFallback { + self.logger.info("voicewake runtime detected (text-only fallback) len=\(match.command.count)") + } else { + self.logger.info("voicewake runtime detected len=\(match.command.count)") + } + await self.beginCapture(command: match.command, triggerEndTime: match.triggerEndTime, config: config) + } else if !transcript.isEmpty, update.error == nil { + if self.isTriggerOnly(transcript: transcript, triggers: config.triggers) { + self.preDetectTask?.cancel() + self.preDetectTask = nil + self.scheduleTriggerOnlyPauseCheck(triggers: config.triggers, config: config) + } else { + self.triggerOnlyTask?.cancel() + self.triggerOnlyTask = nil + self.schedulePreDetectSilenceCheck( + triggers: config.triggers, + gateConfig: gateConfig, + config: config) + } + } + } + + private func maybeLogRecognition( + transcript: String, + segments: [WakeWordSegment], + triggers: [String], + isFinal: Bool, + match: WakeWordGateMatch?, + usedFallback: Bool, + capturing: Bool) + { + guard !transcript.isEmpty else { return } + let level = self.logger.logLevel + guard level == .debug || level == .trace else { return } + if transcript == self.lastLoggedText, !isFinal { + if let last = self.lastLoggedAt, Date().timeIntervalSince(last) < 0.25 { + return + } + } + self.lastLoggedText = transcript + self.lastLoggedAt = Date() + + let textOnly = WakeWordGate.matchesTextOnly(text: transcript, triggers: triggers) + let timingCount = segments.count(where: { $0.start > 0 || $0.duration > 0 }) + let matchSummary = match.map { + "match=true gap=\(String(format: "%.2f", $0.postGap))s cmdLen=\($0.command.count)" + } ?? "match=false" + let segmentSummary = segments.map { seg in + let start = String(format: "%.2f", seg.start) + let end = String(format: "%.2f", seg.end) + return "\(seg.text)@\(start)-\(end)" + }.joined(separator: ", ") + + self.logger.debug( + "voicewake runtime transcript='\(transcript, privacy: .private)' textOnly=\(textOnly) " + + "isFinal=\(isFinal) timing=\(timingCount)/\(segments.count) " + + "capturing=\(capturing) fallback=\(usedFallback) " + + "\(matchSummary) segments=[\(segmentSummary, privacy: .private)]") + } + + private func noteAudioTap(rms: Double) { + let now = Date() + if let last = self.lastTapLogAt, now.timeIntervalSince(last) < 1.0 { + return + } + self.lastTapLogAt = now + let db = 20 * log10(max(rms, 1e-7)) + self.logger.debug( + "voicewake runtime audio tap rms=\(String(format: "%.6f", rms)) " + + "db=\(String(format: "%.1f", db)) capturing=\(self.isCapturing)") + } + + private func noteRecognitionCallback(transcript: String?, isFinal: Bool, error: Error?) { + guard transcript?.isEmpty ?? true else { return } + let now = Date() + if let last = self.lastCallbackLogAt, now.timeIntervalSince(last) < 1.0 { + return + } + self.lastCallbackLogAt = now + let errorSummary = error?.localizedDescription ?? "none" + self.logger.debug( + "voicewake runtime callback empty transcript isFinal=\(isFinal) error=\(errorSummary, privacy: .public)") + } + + private func scheduleTriggerOnlyPauseCheck(triggers: [String], config: RuntimeConfig) { + self.triggerOnlyTask?.cancel() + let lastSeenAt = self.lastTranscriptAt + let lastText = self.lastTranscript + let windowNanos = UInt64(self.triggerPauseWindow * 1_000_000_000) + self.triggerOnlyTask = Task { [weak self, lastSeenAt, lastText] in + try? await Task.sleep(nanoseconds: windowNanos) + guard let self else { return } + await self.triggerOnlyPauseCheck( + lastSeenAt: lastSeenAt, + lastText: lastText, + triggers: triggers, + config: config) + } + } + + private func schedulePreDetectSilenceCheck( + triggers: [String], + gateConfig: WakeWordGateConfig, + config: RuntimeConfig) + { + self.preDetectTask?.cancel() + let lastSeenAt = self.lastTranscriptAt + let lastText = self.lastTranscript + let windowNanos = UInt64(self.preDetectSilenceWindow * 1_000_000_000) + self.preDetectTask = Task { [weak self, lastSeenAt, lastText] in + try? await Task.sleep(nanoseconds: windowNanos) + guard let self else { return } + await self.preDetectSilenceCheck( + lastSeenAt: lastSeenAt, + lastText: lastText, + triggers: triggers, + gateConfig: gateConfig, + config: config) + } + } + + private func triggerOnlyPauseCheck( + lastSeenAt: Date?, + lastText: String?, + triggers: [String], + config: RuntimeConfig) async + { + guard !Task.isCancelled else { return } + guard !self.isCapturing else { return } + guard let lastSeenAt, let lastText else { return } + guard self.lastTranscriptAt == lastSeenAt, self.lastTranscript == lastText else { return } + guard self.isTriggerOnly(transcript: lastText, triggers: triggers) else { return } + if let cooldown = self.cooldownUntil, Date() < cooldown { + return + } + self.logger.info("voicewake runtime detected (trigger-only pause)") + await self.beginCapture(command: "", triggerEndTime: nil, config: config) + } + + private func textOnlyFallbackMatch( + transcript: String, + triggers: [String], + config: WakeWordGateConfig) -> WakeWordGateMatch? + { + guard let command = VoiceWakeTextUtils.textOnlyCommand( + transcript: transcript, + triggers: triggers, + minCommandLength: config.minCommandLength, + trimWake: Self.trimmedAfterTrigger) + else { return nil } + return WakeWordGateMatch(triggerEndTime: 0, postGap: 0, command: command) + } + + private func isTriggerOnly(transcript: String, triggers: [String]) -> Bool { + guard WakeWordGate.matchesTextOnly(text: transcript, triggers: triggers) else { return false } + guard VoiceWakeTextUtils.startsWithTrigger(transcript: transcript, triggers: triggers) else { return false } + return Self.trimmedAfterTrigger(transcript, triggers: triggers).isEmpty + } + + private func preDetectSilenceCheck( + lastSeenAt: Date?, + lastText: String?, + triggers: [String], + gateConfig: WakeWordGateConfig, + config: RuntimeConfig) async + { + guard !Task.isCancelled else { return } + guard !self.isCapturing else { return } + guard let lastSeenAt, let lastText else { return } + guard self.lastTranscriptAt == lastSeenAt, self.lastTranscript == lastText else { return } + guard let match = self.textOnlyFallbackMatch( + transcript: lastText, + triggers: triggers, + config: gateConfig) + else { return } + if let cooldown = self.cooldownUntil, Date() < cooldown { + return + } + self.logger.info("voicewake runtime detected (silence fallback) len=\(match.command.count)") + await self.beginCapture( + command: match.command, + triggerEndTime: match.triggerEndTime, + config: config) + } + + private func beginCapture(command: String, triggerEndTime: TimeInterval?, config: RuntimeConfig) async { + self.listeningState = .voiceWake + self.isCapturing = true + DiagnosticsFileLog.shared.log(category: "voicewake.runtime", event: "beginCapture") + self.capturedTranscript = command + self.committedTranscript = "" + self.volatileTranscript = command + self.captureStartedAt = Date() + self.cooldownUntil = nil + self.heardBeyondTrigger = !command.isEmpty + self.triggerChimePlayed = false + self.activeTriggerEndTime = triggerEndTime + self.preDetectTask?.cancel() + self.preDetectTask = nil + self.triggerOnlyTask?.cancel() + self.triggerOnlyTask = nil + + if config.triggerChime != .none, !self.triggerChimePlayed { + self.triggerChimePlayed = true + await MainActor.run { VoiceWakeChimePlayer.play(config.triggerChime, reason: "voicewake.trigger") } + } + + let snapshot = self.committedTranscript + self.volatileTranscript + let attributed = Self.makeAttributed( + committed: self.committedTranscript, + volatile: self.volatileTranscript, + isFinal: false) + self.overlayToken = await MainActor.run { + VoiceSessionCoordinator.shared.startSession( + source: .wakeWord, + text: snapshot, + attributed: attributed, + forwardEnabled: true) + } + + // Keep the "ears" boosted for the capture window so the status icon animates while recording. + await MainActor.run { AppStateStore.shared.triggerVoiceEars(ttl: nil) } + + self.captureTask?.cancel() + self.captureTask = Task { [weak self] in + guard let self else { return } + await self.monitorCapture(config: config) + } + } + + private func monitorCapture(config: RuntimeConfig) async { + let start = self.captureStartedAt ?? Date() + let hardStop = start.addingTimeInterval(self.captureHardStop) + + while self.isCapturing { + let now = Date() + if now >= hardStop { + // Hard-stop after a maximum duration so we never leave the recognizer pinned open. + await self.finalizeCapture(config: config) + return + } + + let silenceThreshold = self.heardBeyondTrigger ? self.silenceWindow : self.triggerOnlySilenceWindow + if let last = self.lastHeard, now.timeIntervalSince(last) >= silenceThreshold { + await self.finalizeCapture(config: config) + return + } + + try? await Task.sleep(nanoseconds: 200_000_000) + } + } + + private func finalizeCapture(config: RuntimeConfig) async { + guard self.isCapturing else { return } + self.isCapturing = false + // Disarm trigger matching immediately (before halting recognition) to avoid double-trigger + // races from late callbacks that arrive after isCapturing is cleared. + self.cooldownUntil = Date().addingTimeInterval(self.debounceAfterSend) + self.captureTask?.cancel() + self.captureTask = nil + + let finalTranscript = self.capturedTranscript.trimmingCharacters(in: .whitespacesAndNewlines) + DiagnosticsFileLog.shared.log(category: "voicewake.runtime", event: "finalizeCapture", fields: [ + "finalLen": "\(finalTranscript.count)", + ]) + // Stop further recognition events so we don't retrigger immediately with buffered audio. + self.haltRecognitionPipeline() + self.capturedTranscript = "" + self.captureStartedAt = nil + self.lastHeard = nil + self.heardBeyondTrigger = false + self.triggerChimePlayed = false + self.activeTriggerEndTime = nil + self.lastTranscript = nil + self.lastTranscriptAt = nil + self.preDetectTask?.cancel() + self.preDetectTask = nil + self.triggerOnlyTask?.cancel() + self.triggerOnlyTask = nil + + await MainActor.run { AppStateStore.shared.stopVoiceEars() } + if let token = self.overlayToken { + await MainActor.run { VoiceSessionCoordinator.shared.updateLevel(token: token, 0) } + } + + let delay: TimeInterval = 0.0 + let sendChime = finalTranscript.isEmpty ? .none : config.sendChime + if let token = self.overlayToken { + await MainActor.run { + VoiceSessionCoordinator.shared.finalize( + token: token, + text: finalTranscript, + sendChime: sendChime, + autoSendAfter: delay) + } + } else if !finalTranscript.isEmpty { + if sendChime != .none { + await MainActor.run { VoiceWakeChimePlayer.play(sendChime, reason: "voicewake.send") } + } + Task.detached { + await VoiceWakeForwarder.forward(transcript: finalTranscript) + } + } + self.overlayToken = nil + self.scheduleRestartRecognizer() + } + + // MARK: - Audio level handling + + private func noteAudioLevel(rms: Double) { + guard self.isCapturing else { return } + + // Update adaptive noise floor: faster when lower energy (quiet), slower when loud. + let alpha: Double = rms < self.noiseFloorRMS ? 0.08 : 0.01 + self.noiseFloorRMS = max(1e-7, self.noiseFloorRMS + (rms - self.noiseFloorRMS) * alpha) + + let threshold = max(self.minSpeechRMS, self.noiseFloorRMS * self.speechBoostFactor) + if rms >= threshold { + self.lastHeard = Date() + } + + // Normalize against the adaptive threshold so the UI meter stays roughly 0...1 across devices. + let clamped = min(1.0, max(0.0, rms / max(self.minSpeechRMS, threshold))) + if let token = self.overlayToken { + Task { @MainActor in + VoiceSessionCoordinator.shared.updateLevel(token: token, clamped) + } + } + } + + private static func rmsLevel(buffer: AVAudioPCMBuffer) -> Double? { + guard let channelData = buffer.floatChannelData?.pointee else { return nil } + let frameCount = Int(buffer.frameLength) + guard frameCount > 0 else { return nil } + var sum: Double = 0 + for i in 0.. String { + let lower = text.lowercased() + for trigger in triggers { + let token = trigger.lowercased().trimmingCharacters(in: .whitespacesAndNewlines) + guard !token.isEmpty, let range = lower.range(of: token) else { continue } + let after = range.upperBound + let trimmed = text[after...].trimmingCharacters(in: .whitespacesAndNewlines) + return String(trimmed) + } + return text + } + + private static func commandAfterTrigger( + transcript: String, + segments: [WakeWordSegment], + triggerEndTime: TimeInterval?, + triggers: [String]) -> String + { + guard let triggerEndTime else { + return self.trimmedAfterTrigger(transcript, triggers: triggers) + } + let trimmed = WakeWordGate.commandText( + transcript: transcript, + segments: segments, + triggerEndTime: triggerEndTime) + return trimmed.isEmpty ? self.trimmedAfterTrigger(transcript, triggers: triggers) : trimmed + } + + #if DEBUG + static func _testTrimmedAfterTrigger(_ text: String, triggers: [String]) -> String { + self.trimmedAfterTrigger(text, triggers: triggers) + } + + static func _testHasContentAfterTrigger(_ text: String, triggers: [String]) -> Bool { + !self.trimmedAfterTrigger(text, triggers: triggers).isEmpty + } + + static func _testAttributedColor(isFinal: Bool) -> NSColor { + self.makeAttributed(committed: "sample", volatile: "", isFinal: isFinal) + .attribute(.foregroundColor, at: 0, effectiveRange: nil) as? NSColor ?? .clear + } + + #endif + + private static func delta(after committed: String, current: String) -> String { + if current.hasPrefix(committed) { + let start = current.index(current.startIndex, offsetBy: committed.count) + return String(current[start...]) + } + return current + } + + private static func makeAttributed(committed: String, volatile: String, isFinal: Bool) -> NSAttributedString { + let full = NSMutableAttributedString() + let committedAttr: [NSAttributedString.Key: Any] = [ + .foregroundColor: NSColor.labelColor, + .font: NSFont.systemFont(ofSize: 13, weight: .regular), + ] + full.append(NSAttributedString(string: committed, attributes: committedAttr)) + let volatileColor: NSColor = isFinal ? .labelColor : NSColor.tertiaryLabelColor + let volatileAttr: [NSAttributedString.Key: Any] = [ + .foregroundColor: volatileColor, + .font: NSFont.systemFont(ofSize: 13, weight: .regular), + ] + full.append(NSAttributedString(string: volatile, attributes: volatileAttr)) + return full + } +} diff --git a/apps/macos/Sources/OpenClaw/VoiceWakeSettings.swift b/apps/macos/Sources/OpenClaw/VoiceWakeSettings.swift new file mode 100644 index 0000000000000000000000000000000000000000..ca4f4a203553e27b12c31811aa9b7174d645b749 --- /dev/null +++ b/apps/macos/Sources/OpenClaw/VoiceWakeSettings.swift @@ -0,0 +1,673 @@ +import AppKit +import AVFoundation +import Observation +import Speech +import SwabbleKit +import SwiftUI +import UniformTypeIdentifiers + +struct VoiceWakeSettings: View { + @Bindable var state: AppState + let isActive: Bool + @State private var testState: VoiceWakeTestState = .idle + @State private var tester = VoiceWakeTester() + @State private var isTesting = false + @State private var testTimeoutTask: Task? + @State private var availableMics: [AudioInputDevice] = [] + @State private var loadingMics = false + @State private var meterLevel: Double = 0 + @State private var meterError: String? + private let meter = MicLevelMonitor() + @State private var micObserver = AudioInputDeviceObserver() + @State private var micRefreshTask: Task? + @State private var availableLocales: [Locale] = [] + @State private var triggerEntries: [TriggerEntry] = [] + private let fieldLabelWidth: CGFloat = 140 + private let controlWidth: CGFloat = 240 + private let isPreview = ProcessInfo.processInfo.isPreview + + private struct AudioInputDevice: Identifiable, Equatable { + let uid: String + let name: String + var id: String { self.uid } + } + + private struct TriggerEntry: Identifiable { + let id: UUID + var value: String + } + + private var voiceWakeBinding: Binding { + Binding( + get: { self.state.swabbleEnabled }, + set: { newValue in + Task { await self.state.setVoiceWakeEnabled(newValue) } + }) + } + + var body: some View { + ScrollView(.vertical) { + VStack(alignment: .leading, spacing: 14) { + SettingsToggleRow( + title: "Enable Voice Wake", + subtitle: "Listen for a wake phrase (e.g. \"Claude\") before running voice commands. " + + "Voice recognition runs fully on-device.", + binding: self.voiceWakeBinding) + .disabled(!voiceWakeSupported) + + SettingsToggleRow( + title: "Hold Right Option to talk", + subtitle: """ + Push-to-talk mode that starts listening while you hold the key + and shows the preview overlay. + """, + binding: self.$state.voicePushToTalkEnabled) + .disabled(!voiceWakeSupported) + + if !voiceWakeSupported { + Label("Voice Wake requires macOS 26 or newer.", systemImage: "exclamationmark.triangle.fill") + .font(.callout) + .foregroundStyle(.yellow) + .padding(8) + .background(Color.secondary.opacity(0.15)) + .clipShape(RoundedRectangle(cornerRadius: 8)) + } + + self.localePicker + self.micPicker + self.levelMeter + + VoiceWakeTestCard( + testState: self.$testState, + isTesting: self.$isTesting, + onToggle: self.toggleTest) + + self.chimeSection + + self.triggerTable + + Spacer(minLength: 8) + } + .frame(maxWidth: .infinity, alignment: .leading) + .padding(.horizontal, 12) + } + .task { + guard !self.isPreview else { return } + await self.loadMicsIfNeeded() + } + .task { + guard !self.isPreview else { return } + await self.loadLocalesIfNeeded() + } + .task { + guard !self.isPreview else { return } + await self.restartMeter() + } + .onAppear { + guard !self.isPreview else { return } + self.startMicObserver() + self.loadTriggerEntries() + } + .onChange(of: self.state.voiceWakeMicID) { _, _ in + guard !self.isPreview else { return } + self.updateSelectedMicName() + Task { await self.restartMeter() } + } + .onChange(of: self.isActive) { _, active in + guard !self.isPreview else { return } + if !active { + self.tester.stop() + self.isTesting = false + self.testState = .idle + self.testTimeoutTask?.cancel() + self.micRefreshTask?.cancel() + self.micRefreshTask = nil + Task { await self.meter.stop() } + self.micObserver.stop() + self.syncTriggerEntriesToState() + } else { + self.startMicObserver() + self.loadTriggerEntries() + } + } + .onDisappear { + guard !self.isPreview else { return } + self.tester.stop() + self.isTesting = false + self.testState = .idle + self.testTimeoutTask?.cancel() + self.micRefreshTask?.cancel() + self.micRefreshTask = nil + self.micObserver.stop() + Task { await self.meter.stop() } + self.syncTriggerEntriesToState() + } + } + + private func loadTriggerEntries() { + self.triggerEntries = self.state.swabbleTriggerWords.map { TriggerEntry(id: UUID(), value: $0) } + } + + private func syncTriggerEntriesToState() { + self.state.swabbleTriggerWords = self.triggerEntries.map(\.value) + } + + private var triggerTable: some View { + VStack(alignment: .leading, spacing: 8) { + HStack { + Text("Trigger words") + .font(.callout.weight(.semibold)) + Spacer() + Button { + self.addWord() + } label: { + Label("Add word", systemImage: "plus") + } + .disabled(self.triggerEntries + .contains(where: { $0.value.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty })) + + Button("Reset defaults") { + self.triggerEntries = defaultVoiceWakeTriggers.map { TriggerEntry(id: UUID(), value: $0) } + self.syncTriggerEntriesToState() + } + } + + VStack(spacing: 0) { + ForEach(self.$triggerEntries) { $entry in + HStack(spacing: 8) { + TextField("Wake word", text: $entry.value) + .textFieldStyle(.roundedBorder) + .onSubmit { + self.syncTriggerEntriesToState() + } + + Button { + self.removeWord(id: entry.id) + } label: { + Image(systemName: "trash") + } + .buttonStyle(.borderless) + .help("Remove trigger word") + .frame(width: 24) + } + .padding(8) + + if entry.id != self.triggerEntries.last?.id { + Divider() + } + } + } + .frame(maxWidth: .infinity, minHeight: 180, alignment: .topLeading) + .background(Color(nsColor: .textBackgroundColor)) + .clipShape(RoundedRectangle(cornerRadius: 6)) + .overlay( + RoundedRectangle(cornerRadius: 6) + .stroke(Color.secondary.opacity(0.25), lineWidth: 1)) + + Text( + "OpenClaw reacts when any trigger appears in a transcription. " + + "Keep them short to avoid false positives.") + .font(.footnote) + .foregroundStyle(.secondary) + .fixedSize(horizontal: false, vertical: true) + } + } + + private var chimeSection: some View { + VStack(alignment: .leading, spacing: 10) { + HStack(alignment: .firstTextBaseline, spacing: 10) { + Text("Sounds") + .font(.callout.weight(.semibold)) + Spacer() + } + + self.chimeRow( + title: "Trigger sound", + selection: self.$state.voiceWakeTriggerChime) + + self.chimeRow( + title: "Send sound", + selection: self.$state.voiceWakeSendChime) + } + .padding(.top, 4) + } + + private func addWord() { + self.triggerEntries.append(TriggerEntry(id: UUID(), value: "")) + } + + private func removeWord(id: UUID) { + self.triggerEntries.removeAll { $0.id == id } + self.syncTriggerEntriesToState() + } + + private func toggleTest() { + guard voiceWakeSupported else { + self.testState = .failed("Voice Wake requires macOS 26 or newer.") + return + } + if self.isTesting { + self.tester.finalize() + self.isTesting = false + self.testState = .finalizing + Task { @MainActor in + try? await Task.sleep(nanoseconds: 2_000_000_000) + if self.testState == .finalizing { + self.tester.stop() + self.testState = .failed("Stopped") + } + } + self.testTimeoutTask?.cancel() + return + } + + let triggers = self.sanitizedTriggers() + self.tester.stop() + self.testTimeoutTask?.cancel() + self.isTesting = true + self.testState = .requesting + Task { @MainActor in + do { + try await self.tester.start( + triggers: triggers, + micID: self.state.voiceWakeMicID.isEmpty ? nil : self.state.voiceWakeMicID, + localeID: self.state.voiceWakeLocaleID, + onUpdate: { newState in + DispatchQueue.main.async { [self] in + self.testState = newState + if case .detected = newState { self.isTesting = false } + if case .failed = newState { self.isTesting = false } + if case .detected = newState { self.testTimeoutTask?.cancel() } + if case .failed = newState { self.testTimeoutTask?.cancel() } + } + }) + self.testTimeoutTask?.cancel() + self.testTimeoutTask = Task { @MainActor in + try? await Task.sleep(nanoseconds: 10 * 1_000_000_000) + guard !Task.isCancelled else { return } + if self.isTesting { + self.tester.stop() + if case let .hearing(text) = self.testState, + let command = Self.textOnlyCommand(from: text, triggers: triggers) + { + self.testState = .detected(command) + } else { + self.testState = .failed("Timeout: no trigger heard") + } + self.isTesting = false + } + } + } catch { + self.tester.stop() + self.testState = .failed(error.localizedDescription) + self.isTesting = false + self.testTimeoutTask?.cancel() + } + } + } + + private func chimeRow(title: String, selection: Binding) -> some View { + HStack(alignment: .center, spacing: 10) { + Text(title) + .font(.callout.weight(.semibold)) + .frame(width: self.fieldLabelWidth, alignment: .leading) + + Menu { + Button("No Sound") { self.selectChime(.none, binding: selection) } + Divider() + ForEach(VoiceWakeChimeCatalog.systemOptions, id: \.self) { option in + Button(VoiceWakeChimeCatalog.displayName(for: option)) { + self.selectChime(.system(name: option), binding: selection) + } + } + Divider() + Button("Choose file…") { self.chooseCustomChime(for: selection) } + } label: { + HStack(spacing: 6) { + Text(selection.wrappedValue.displayLabel) + .lineLimit(1) + .truncationMode(.middle) + Spacer() + Image(systemName: "chevron.down") + .font(.caption) + .foregroundStyle(.secondary) + } + .padding(6) + .frame(minWidth: self.controlWidth, maxWidth: .infinity, alignment: .leading) + .background(Color(nsColor: .windowBackgroundColor)) + .overlay( + RoundedRectangle(cornerRadius: 6) + .stroke(Color.secondary.opacity(0.25), lineWidth: 1)) + .clipShape(RoundedRectangle(cornerRadius: 6)) + } + + Button("Play") { + VoiceWakeChimePlayer.play(selection.wrappedValue) + } + .keyboardShortcut(.space, modifiers: [.command]) + } + } + + private func chooseCustomChime(for selection: Binding) { + let panel = NSOpenPanel() + panel.allowedContentTypes = [.audio] + panel.allowsMultipleSelection = false + panel.canChooseDirectories = false + panel.resolvesAliases = true + panel.begin { response in + guard response == .OK, let url = panel.url else { return } + do { + let bookmark = try url.bookmarkData( + options: [.withSecurityScope], + includingResourceValuesForKeys: nil, + relativeTo: nil) + let chosen = VoiceWakeChime.custom(displayName: url.lastPathComponent, bookmark: bookmark) + selection.wrappedValue = chosen + VoiceWakeChimePlayer.play(chosen) + } catch { + // Ignore failures; user can retry. + } + } + } + + private func selectChime(_ chime: VoiceWakeChime, binding: Binding) { + binding.wrappedValue = chime + VoiceWakeChimePlayer.play(chime) + } + + private func sanitizedTriggers() -> [String] { + sanitizeVoiceWakeTriggers(self.state.swabbleTriggerWords) + } + + private static func textOnlyCommand(from transcript: String, triggers: [String]) -> String? { + VoiceWakeTextUtils.textOnlyCommand( + transcript: transcript, + triggers: triggers, + minCommandLength: 1, + trimWake: { WakeWordGate.stripWake(text: $0, triggers: $1) }) + } + + private var micPicker: some View { + VStack(alignment: .leading, spacing: 6) { + HStack(alignment: .firstTextBaseline, spacing: 10) { + Text("Microphone") + .font(.callout.weight(.semibold)) + .frame(width: self.fieldLabelWidth, alignment: .leading) + Picker("Microphone", selection: self.$state.voiceWakeMicID) { + Text("System default").tag("") + if self.isSelectedMicUnavailable { + Text(self.state.voiceWakeMicName.isEmpty ? "Unavailable" : self.state.voiceWakeMicName) + .tag(self.state.voiceWakeMicID) + } + ForEach(self.availableMics) { mic in + Text(mic.name).tag(mic.uid) + } + } + .labelsHidden() + .frame(width: self.controlWidth) + } + if self.isSelectedMicUnavailable { + HStack(spacing: 10) { + Color.clear.frame(width: self.fieldLabelWidth, height: 1) + Text("Disconnected (using System default)") + .font(.caption) + .foregroundStyle(.secondary) + .lineLimit(1) + } + } + if self.loadingMics { + ProgressView().controlSize(.small) + } + } + } + + private var localePicker: some View { + VStack(alignment: .leading, spacing: 6) { + HStack(alignment: .firstTextBaseline, spacing: 10) { + Text("Recognition language") + .font(.callout.weight(.semibold)) + .frame(width: self.fieldLabelWidth, alignment: .leading) + Picker("Language", selection: self.$state.voiceWakeLocaleID) { + let current = Locale(identifier: Locale.current.identifier) + Text("\(self.friendlyName(for: current)) (System)").tag(Locale.current.identifier) + ForEach(self.availableLocales.map(\.identifier), id: \.self) { id in + if id != Locale.current.identifier { + Text(self.friendlyName(for: Locale(identifier: id))).tag(id) + } + } + } + .labelsHidden() + .frame(width: self.controlWidth) + } + + if !self.state.voiceWakeAdditionalLocaleIDs.isEmpty { + VStack(alignment: .leading, spacing: 8) { + Text("Additional languages") + .font(.footnote.weight(.semibold)) + ForEach( + Array(self.state.voiceWakeAdditionalLocaleIDs.enumerated()), + id: \.offset) + { idx, localeID in + HStack(spacing: 8) { + Picker("Extra \(idx + 1)", selection: Binding( + get: { localeID }, + set: { newValue in + guard self.state + .voiceWakeAdditionalLocaleIDs.indices + .contains(idx) else { return } + self.state + .voiceWakeAdditionalLocaleIDs[idx] = + newValue + })) { + ForEach(self.availableLocales.map(\.identifier), id: \.self) { id in + Text(self.friendlyName(for: Locale(identifier: id))).tag(id) + } + } + .labelsHidden() + .frame(width: 220) + + Button { + guard self.state.voiceWakeAdditionalLocaleIDs.indices.contains(idx) else { return } + self.state.voiceWakeAdditionalLocaleIDs.remove(at: idx) + } label: { + Image(systemName: "trash") + } + .buttonStyle(.borderless) + .help("Remove language") + } + } + + Button { + if let first = availableLocales.first { + self.state.voiceWakeAdditionalLocaleIDs.append(first.identifier) + } + } label: { + Label("Add language", systemImage: "plus") + } + .disabled(self.availableLocales.isEmpty) + } + .padding(.top, 4) + } else { + Button { + if let first = availableLocales.first { + self.state.voiceWakeAdditionalLocaleIDs.append(first.identifier) + } + } label: { + Label("Add additional language", systemImage: "plus") + } + .buttonStyle(.link) + .disabled(self.availableLocales.isEmpty) + .padding(.top, 4) + } + + Text("Languages are tried in order. Models may need a first-use download on macOS 26.") + .font(.caption) + .foregroundStyle(.secondary) + } + } + + @MainActor + private func loadMicsIfNeeded(force: Bool = false) async { + guard force || self.availableMics.isEmpty, !self.loadingMics else { return } + self.loadingMics = true + let discovery = AVCaptureDevice.DiscoverySession( + deviceTypes: [.external, .microphone], + mediaType: .audio, + position: .unspecified) + let aliveUIDs = AudioInputDeviceObserver.aliveInputDeviceUIDs() + let connectedDevices = discovery.devices.filter(\.isConnected) + let devices = aliveUIDs.isEmpty + ? connectedDevices + : connectedDevices.filter { aliveUIDs.contains($0.uniqueID) } + self.availableMics = devices.map { AudioInputDevice(uid: $0.uniqueID, name: $0.localizedName) } + self.updateSelectedMicName() + self.loadingMics = false + } + + private var isSelectedMicUnavailable: Bool { + let selected = self.state.voiceWakeMicID + guard !selected.isEmpty else { return false } + return !self.availableMics.contains(where: { $0.uid == selected }) + } + + @MainActor + private func updateSelectedMicName() { + let selected = self.state.voiceWakeMicID + if selected.isEmpty { + self.state.voiceWakeMicName = "" + return + } + if let match = self.availableMics.first(where: { $0.uid == selected }) { + self.state.voiceWakeMicName = match.name + } + } + + private func startMicObserver() { + self.micObserver.start { + Task { @MainActor in + self.scheduleMicRefresh() + } + } + } + + @MainActor + private func scheduleMicRefresh() { + self.micRefreshTask?.cancel() + self.micRefreshTask = Task { @MainActor in + try? await Task.sleep(nanoseconds: 300_000_000) + guard !Task.isCancelled else { return } + await self.loadMicsIfNeeded(force: true) + await self.restartMeter() + } + } + + @MainActor + private func loadLocalesIfNeeded() async { + guard self.availableLocales.isEmpty else { return } + self.availableLocales = Array(SFSpeechRecognizer.supportedLocales()).sorted { lhs, rhs in + self.friendlyName(for: lhs) + .localizedCaseInsensitiveCompare(self.friendlyName(for: rhs)) == .orderedAscending + } + } + + private func friendlyName(for locale: Locale) -> String { + let cleanedID = normalizeLocaleIdentifier(locale.identifier) + let cleanLocale = Locale(identifier: cleanedID) + + if let langCode = cleanLocale.language.languageCode?.identifier, + let lang = cleanLocale.localizedString(forLanguageCode: langCode), + let regionCode = cleanLocale.region?.identifier, + let region = cleanLocale.localizedString(forRegionCode: regionCode) + { + return "\(lang) (\(region))" + } + if let langCode = cleanLocale.language.languageCode?.identifier, + let lang = cleanLocale.localizedString(forLanguageCode: langCode) + { + return lang + } + return cleanLocale.localizedString(forIdentifier: cleanedID) ?? cleanedID + } + + private var levelMeter: some View { + VStack(alignment: .leading, spacing: 6) { + HStack(alignment: .center, spacing: 10) { + Text("Live level") + .font(.callout.weight(.semibold)) + .frame(width: self.fieldLabelWidth, alignment: .leading) + MicLevelBar(level: self.meterLevel) + .frame(width: self.controlWidth, alignment: .leading) + Text(self.levelLabel) + .font(.callout.monospacedDigit()) + .foregroundStyle(.secondary) + .frame(width: 60, alignment: .trailing) + } + if let meterError { + Text(meterError) + .font(.footnote) + .foregroundStyle(.secondary) + } + } + } + + private var levelLabel: String { + let db = (meterLevel * 50) - 50 + return String(format: "%.0f dB", db) + } + + @MainActor + private func restartMeter() async { + self.meterError = nil + await self.meter.stop() + do { + try await self.meter.start { [weak state] level in + Task { @MainActor in + guard state != nil else { return } + self.meterLevel = level + } + } + } catch { + self.meterError = error.localizedDescription + } + } +} + +#if DEBUG +struct VoiceWakeSettings_Previews: PreviewProvider { + static var previews: some View { + VoiceWakeSettings(state: .preview, isActive: true) + .frame(width: SettingsTab.windowWidth, height: SettingsTab.windowHeight) + } +} + +@MainActor +extension VoiceWakeSettings { + static func exerciseForTesting() { + let state = AppState(preview: true) + state.swabbleEnabled = true + state.voicePushToTalkEnabled = true + state.swabbleTriggerWords = ["Claude", "Hey"] + + let view = VoiceWakeSettings(state: state, isActive: true) + view.availableMics = [AudioInputDevice(uid: "mic-1", name: "Built-in")] + view.availableLocales = [Locale(identifier: "en_US")] + view.meterLevel = 0.42 + view.meterError = "No input" + view.testState = .detected("ok") + view.isTesting = true + view.triggerEntries = [TriggerEntry(id: UUID(), value: "Claude")] + + _ = view.body + _ = view.localePicker + _ = view.micPicker + _ = view.levelMeter + _ = view.triggerTable + _ = view.chimeSection + + view.addWord() + if let entryId = view.triggerEntries.first?.id { + view.removeWord(id: entryId) + } + } +} +#endif diff --git a/apps/macos/Sources/OpenClaw/VoiceWakeTestCard.swift b/apps/macos/Sources/OpenClaw/VoiceWakeTestCard.swift new file mode 100644 index 0000000000000000000000000000000000000000..7de20885a6c8885718550857904545bdf164455d --- /dev/null +++ b/apps/macos/Sources/OpenClaw/VoiceWakeTestCard.swift @@ -0,0 +1,95 @@ +import SwiftUI + +struct VoiceWakeTestCard: View { + @Binding var testState: VoiceWakeTestState + @Binding var isTesting: Bool + let onToggle: () -> Void + + var body: some View { + VStack(alignment: .leading, spacing: 10) { + HStack { + Text("Test Voice Wake") + .font(.callout.weight(.semibold)) + Spacer() + Button(action: self.onToggle) { + Label( + self.isTesting ? "Stop" : "Start test", + systemImage: self.isTesting ? "stop.circle.fill" : "play.circle") + } + .buttonStyle(.borderedProminent) + .tint(self.isTesting ? .red : .accentColor) + } + + HStack(spacing: 8) { + self.statusIcon + VStack(alignment: .leading, spacing: 4) { + Text(self.statusText) + .font(.subheadline) + .frame(maxHeight: 22, alignment: .center) + if case let .detected(text) = testState { + Text("Heard: \(text)") + .font(.footnote) + .foregroundStyle(.secondary) + .lineLimit(2) + } + } + Spacer() + } + .padding(10) + .background(.quaternary.opacity(0.2)) + .clipShape(RoundedRectangle(cornerRadius: 8)) + .frame(minHeight: 54) + } + .padding(.vertical, 2) + } + + private var statusIcon: some View { + switch self.testState { + case .idle: + AnyView(Image(systemName: "waveform").foregroundStyle(.secondary)) + + case .requesting: + AnyView(ProgressView().controlSize(.small)) + + case .listening, .hearing: + AnyView( + Image(systemName: "ear.and.waveform") + .symbolEffect(.pulse) + .foregroundStyle(Color.accentColor)) + + case .finalizing: + AnyView(ProgressView().controlSize(.small)) + + case .detected: + AnyView(Image(systemName: "checkmark.circle.fill").foregroundStyle(.green)) + + case .failed: + AnyView(Image(systemName: "exclamationmark.triangle.fill").foregroundStyle(.yellow)) + } + } + + private var statusText: String { + switch self.testState { + case .idle: + "Press start, say a trigger word, and wait for detection." + + case .requesting: + "Requesting mic & speech permission…" + + case .listening: + "Listening… say your trigger word." + + case let .hearing(text): + "Heard: \(text)" + + case .finalizing: + "Finalizing…" + + case .detected: + "Voice wake detected!" + + case let .failed(reason): + reason + } + } +} diff --git a/apps/macos/Sources/OpenClaw/VoiceWakeTester.swift b/apps/macos/Sources/OpenClaw/VoiceWakeTester.swift new file mode 100644 index 0000000000000000000000000000000000000000..b3d0c58d90c756d5ed6d22c7f176aa94fe92cc64 --- /dev/null +++ b/apps/macos/Sources/OpenClaw/VoiceWakeTester.swift @@ -0,0 +1,473 @@ +import AVFoundation +import Foundation +import Speech +import SwabbleKit + +enum VoiceWakeTestState: Equatable { + case idle + case requesting + case listening + case hearing(String) + case finalizing + case detected(String) + case failed(String) +} + +final class VoiceWakeTester { + private let recognizer: SFSpeechRecognizer? + private var audioEngine: AVAudioEngine? + private var recognitionRequest: SFSpeechAudioBufferRecognitionRequest? + private var recognitionTask: SFSpeechRecognitionTask? + private var isStopping = false + private var isFinalizing = false + private var detectionStart: Date? + private var lastHeard: Date? + private var lastLoggedText: String? + private var lastLoggedAt: Date? + private var lastTranscript: String? + private var lastTranscriptAt: Date? + private var silenceTask: Task? + private var currentTriggers: [String] = [] + private var holdingAfterDetect = false + private var detectedText: String? + private let logger = Logger(subsystem: "ai.openclaw", category: "voicewake") + private let silenceWindow: TimeInterval = 1.0 + + init(locale: Locale = .current) { + self.recognizer = SFSpeechRecognizer(locale: locale) + } + + func start( + triggers: [String], + micID: String?, + localeID: String?, + onUpdate: @escaping @Sendable (VoiceWakeTestState) -> Void) async throws + { + guard self.recognitionTask == nil else { return } + self.isStopping = false + self.isFinalizing = false + self.holdingAfterDetect = false + self.detectedText = nil + self.lastHeard = nil + self.lastLoggedText = nil + self.lastLoggedAt = nil + self.lastTranscript = nil + self.lastTranscriptAt = nil + self.silenceTask?.cancel() + self.silenceTask = nil + self.currentTriggers = triggers + let chosenLocale = localeID.flatMap { Locale(identifier: $0) } ?? Locale.current + let recognizer = SFSpeechRecognizer(locale: chosenLocale) + guard let recognizer, recognizer.isAvailable else { + throw NSError( + domain: "VoiceWakeTester", + code: 1, + userInfo: [NSLocalizedDescriptionKey: "Speech recognition unavailable"]) + } + recognizer.defaultTaskHint = .dictation + + guard Self.hasPrivacyStrings else { + throw NSError( + domain: "VoiceWakeTester", + code: 3, + userInfo: [ + NSLocalizedDescriptionKey: """ + Missing mic/speech privacy strings. Rebuild the mac app (scripts/restart-mac.sh) \ + to include usage descriptions. + """, + ]) + } + + let granted = try await Self.ensurePermissions() + guard granted else { + throw NSError( + domain: "VoiceWakeTester", + code: 2, + userInfo: [NSLocalizedDescriptionKey: "Microphone or speech permission denied"]) + } + + self.logInputSelection(preferredMicID: micID) + self.configureSession(preferredMicID: micID) + + let engine = AVAudioEngine() + self.audioEngine = engine + + self.recognitionRequest = SFSpeechAudioBufferRecognitionRequest() + self.recognitionRequest?.shouldReportPartialResults = true + self.recognitionRequest?.taskHint = .dictation + let request = self.recognitionRequest + + let inputNode = engine.inputNode + let format = inputNode.outputFormat(forBus: 0) + guard format.channelCount > 0, format.sampleRate > 0 else { + self.audioEngine = nil + throw NSError( + domain: "VoiceWakeTester", + code: 4, + userInfo: [NSLocalizedDescriptionKey: "No audio input available"]) + } + inputNode.removeTap(onBus: 0) + inputNode.installTap(onBus: 0, bufferSize: 2048, format: format) { [weak request] buffer, _ in + request?.append(buffer) + } + + engine.prepare() + try engine.start() + DispatchQueue.main.async { + onUpdate(.listening) + } + + self.detectionStart = Date() + self.lastHeard = self.detectionStart + + guard let request = recognitionRequest else { return } + + self.recognitionTask = recognizer.recognitionTask(with: request) { [weak self] result, error in + guard let self, !self.isStopping else { return } + let text = result?.bestTranscription.formattedString ?? "" + let segments = result.map { WakeWordSpeechSegments.from( + transcription: $0.bestTranscription, + transcript: text) } ?? [] + let isFinal = result?.isFinal ?? false + let gateConfig = WakeWordGateConfig(triggers: triggers) + var match = WakeWordGate.match(transcript: text, segments: segments, config: gateConfig) + if match == nil, isFinal { + match = self.textOnlyFallbackMatch( + transcript: text, + triggers: triggers, + config: gateConfig) + } + self.maybeLogDebug( + transcript: text, + segments: segments, + triggers: triggers, + match: match, + isFinal: isFinal) + let errorMessage = error?.localizedDescription + + Task { [weak self] in + guard let self, !self.isStopping else { return } + await self.handleResult( + match: match, + text: text, + isFinal: isFinal, + errorMessage: errorMessage, + onUpdate: onUpdate) + } + } + } + + func stop() { + self.stop(force: true) + } + + func finalize(timeout: TimeInterval = 1.5) { + guard self.recognitionTask != nil else { + self.stop(force: true) + return + } + self.isFinalizing = true + self.recognitionRequest?.endAudio() + if let engine = self.audioEngine { + engine.inputNode.removeTap(onBus: 0) + engine.stop() + } + Task { [weak self] in + guard let self else { return } + try? await Task.sleep(nanoseconds: UInt64(timeout * 1_000_000_000)) + if !self.isStopping { + self.stop(force: true) + } + } + } + + private func stop(force: Bool) { + if force { self.isStopping = true } + self.isFinalizing = false + self.recognitionRequest?.endAudio() + self.recognitionTask?.cancel() + self.recognitionTask = nil + self.recognitionRequest = nil + if let engine = self.audioEngine { + engine.inputNode.removeTap(onBus: 0) + engine.stop() + } + self.audioEngine = nil + self.holdingAfterDetect = false + self.detectedText = nil + self.lastHeard = nil + self.detectionStart = nil + self.lastLoggedText = nil + self.lastLoggedAt = nil + self.lastTranscript = nil + self.lastTranscriptAt = nil + self.silenceTask?.cancel() + self.silenceTask = nil + self.currentTriggers = [] + } + + private func handleResult( + match: WakeWordGateMatch?, + text: String, + isFinal: Bool, + errorMessage: String?, + onUpdate: @escaping @Sendable (VoiceWakeTestState) -> Void) async + { + if !text.isEmpty { + self.lastHeard = Date() + self.lastTranscript = text + self.lastTranscriptAt = Date() + } + if self.holdingAfterDetect { + return + } + if let match, !match.command.isEmpty { + self.holdingAfterDetect = true + self.detectedText = match.command + self.logger.info("voice wake detected (test) (len=\(match.command.count))") + await MainActor.run { AppStateStore.shared.triggerVoiceEars(ttl: nil) } + self.stop() + await MainActor.run { + AppStateStore.shared.stopVoiceEars() + onUpdate(.detected(match.command)) + } + return + } + if !isFinal, !text.isEmpty { + self.scheduleSilenceCheck( + triggers: self.currentTriggers, + onUpdate: onUpdate) + } + if self.isFinalizing { + Task { @MainActor in onUpdate(.finalizing) } + } + if let errorMessage { + self.stop(force: true) + Task { @MainActor in onUpdate(.failed(errorMessage)) } + return + } + if isFinal { + self.stop(force: true) + let state: VoiceWakeTestState = text.isEmpty + ? .failed("No speech detected") + : .failed("No trigger heard: “\(text)”") + Task { @MainActor in onUpdate(state) } + } else { + let state: VoiceWakeTestState = text.isEmpty ? .listening : .hearing(text) + Task { @MainActor in onUpdate(state) } + } + } + + private func maybeLogDebug( + transcript: String, + segments: [WakeWordSegment], + triggers: [String], + match: WakeWordGateMatch?, + isFinal: Bool) + { + guard !transcript.isEmpty else { return } + let level = self.logger.logLevel + guard level == .debug || level == .trace else { return } + if transcript == self.lastLoggedText, !isFinal { + if let last = self.lastLoggedAt, Date().timeIntervalSince(last) < 0.25 { + return + } + } + self.lastLoggedText = transcript + self.lastLoggedAt = Date() + + let textOnly = WakeWordGate.matchesTextOnly(text: transcript, triggers: triggers) + let gaps = Self.debugCandidateGaps(triggers: triggers, segments: segments) + let segmentSummary = Self.debugSegments(segments) + let timingCount = segments.count(where: { $0.start > 0 || $0.duration > 0 }) + let matchSummary = match.map { + "match=true gap=\(String(format: "%.2f", $0.postGap))s cmdLen=\($0.command.count)" + } ?? "match=false" + + self.logger.debug( + "voicewake test transcript='\(transcript, privacy: .private)' textOnly=\(textOnly) " + + "isFinal=\(isFinal) timing=\(timingCount)/\(segments.count) " + + "\(matchSummary) gaps=[\(gaps, privacy: .private)] segments=[\(segmentSummary, privacy: .private)]") + } + + private static func debugSegments(_ segments: [WakeWordSegment]) -> String { + segments.map { seg in + let start = String(format: "%.2f", seg.start) + let end = String(format: "%.2f", seg.end) + return "\(seg.text)@\(start)-\(end)" + }.joined(separator: ", ") + } + + private static func debugCandidateGaps(triggers: [String], segments: [WakeWordSegment]) -> String { + let tokens = self.normalizeSegments(segments) + guard !tokens.isEmpty else { return "" } + let triggerTokens = self.normalizeTriggers(triggers) + var gaps: [String] = [] + + for trigger in triggerTokens { + let count = trigger.tokens.count + guard count > 0, tokens.count > count else { continue } + for i in 0...(tokens.count - count - 1) { + let matched = (0.. [DebugTriggerTokens] { + var output: [DebugTriggerTokens] = [] + for trigger in triggers { + let tokens = trigger + .split(whereSeparator: { $0.isWhitespace }) + .map { VoiceWakeTextUtils.normalizeToken(String($0)) } + .filter { !$0.isEmpty } + if tokens.isEmpty { continue } + output.append(DebugTriggerTokens(tokens: tokens)) + } + return output + } + + private static func normalizeSegments(_ segments: [WakeWordSegment]) -> [DebugToken] { + segments.compactMap { segment in + let normalized = VoiceWakeTextUtils.normalizeToken(segment.text) + guard !normalized.isEmpty else { return nil } + return DebugToken( + normalized: normalized, + start: segment.start, + end: segment.end) + } + } + + private func textOnlyFallbackMatch( + transcript: String, + triggers: [String], + config: WakeWordGateConfig) -> WakeWordGateMatch? + { + guard let command = VoiceWakeTextUtils.textOnlyCommand( + transcript: transcript, + triggers: triggers, + minCommandLength: config.minCommandLength, + trimWake: { WakeWordGate.stripWake(text: $0, triggers: $1) }) + else { return nil } + return WakeWordGateMatch(triggerEndTime: 0, postGap: 0, command: command) + } + + private func holdUntilSilence(onUpdate: @escaping @Sendable (VoiceWakeTestState) -> Void) { + Task { [weak self] in + guard let self else { return } + let detectedAt = Date() + let hardStop = detectedAt.addingTimeInterval(6) // cap overall listen after trigger + + while !self.isStopping { + let now = Date() + if now >= hardStop { break } + if let last = self.lastHeard, now.timeIntervalSince(last) >= silenceWindow { + break + } + try? await Task.sleep(nanoseconds: 200_000_000) + } + if !self.isStopping { + self.stop() + await MainActor.run { AppStateStore.shared.stopVoiceEars() } + if let detectedText { + self.logger.info("voice wake hold finished; len=\(detectedText.count)") + Task { @MainActor in onUpdate(.detected(detectedText)) } + } + } + } + } + + private func scheduleSilenceCheck( + triggers: [String], + onUpdate: @escaping @Sendable (VoiceWakeTestState) -> Void) + { + self.silenceTask?.cancel() + let lastSeenAt = self.lastTranscriptAt + let lastText = self.lastTranscript + self.silenceTask = Task { [weak self] in + guard let self else { return } + try? await Task.sleep(nanoseconds: UInt64(self.silenceWindow * 1_000_000_000)) + guard !Task.isCancelled else { return } + guard !self.isStopping, !self.holdingAfterDetect else { return } + guard let lastSeenAt, let lastText else { return } + guard self.lastTranscriptAt == lastSeenAt, self.lastTranscript == lastText else { return } + guard let match = self.textOnlyFallbackMatch( + transcript: lastText, + triggers: triggers, + config: WakeWordGateConfig(triggers: triggers)) else { return } + self.holdingAfterDetect = true + self.detectedText = match.command + self.logger.info("voice wake detected (test, silence) (len=\(match.command.count))") + await MainActor.run { AppStateStore.shared.triggerVoiceEars(ttl: nil) } + self.stop() + await MainActor.run { + AppStateStore.shared.stopVoiceEars() + onUpdate(.detected(match.command)) + } + } + } + + private func configureSession(preferredMicID: String?) { + _ = preferredMicID + } + + private func logInputSelection(preferredMicID: String?) { + let preferred = (preferredMicID?.isEmpty == false) ? preferredMicID! : "system-default" + self.logger.info( + "voicewake test input preferred=\(preferred, privacy: .public) " + + "\(AudioInputDeviceObserver.defaultInputDeviceSummary(), privacy: .public)") + } + + private nonisolated static func ensurePermissions() async throws -> Bool { + let speechStatus = SFSpeechRecognizer.authorizationStatus() + if speechStatus == .notDetermined { + let granted = await withCheckedContinuation { continuation in + SFSpeechRecognizer.requestAuthorization { status in + continuation.resume(returning: status == .authorized) + } + } + guard granted else { return false } + } else if speechStatus != .authorized { + return false + } + + let micStatus = AVCaptureDevice.authorizationStatus(for: .audio) + switch micStatus { + case .authorized: return true + + case .notDetermined: + return await withCheckedContinuation { continuation in + AVCaptureDevice.requestAccess(for: .audio) { granted in + continuation.resume(returning: granted) + } + } + + default: + return false + } + } + + private static var hasPrivacyStrings: Bool { + let speech = Bundle.main.object(forInfoDictionaryKey: "NSSpeechRecognitionUsageDescription") as? String + let mic = Bundle.main.object(forInfoDictionaryKey: "NSMicrophoneUsageDescription") as? String + return speech?.isEmpty == false && mic?.isEmpty == false + } +} + +extension VoiceWakeTester: @unchecked Sendable {} diff --git a/apps/macos/Sources/OpenClaw/VoiceWakeTextUtils.swift b/apps/macos/Sources/OpenClaw/VoiceWakeTextUtils.swift new file mode 100644 index 0000000000000000000000000000000000000000..9311765ad5c047cedb9197cb6f43a34c650c07c7 --- /dev/null +++ b/apps/macos/Sources/OpenClaw/VoiceWakeTextUtils.swift @@ -0,0 +1,48 @@ +import Foundation +import SwabbleKit + +enum VoiceWakeTextUtils { + private static let whitespaceAndPunctuation = CharacterSet.whitespacesAndNewlines + .union(.punctuationCharacters) + typealias TrimWake = (String, [String]) -> String + + static func normalizeToken(_ token: String) -> String { + token + .trimmingCharacters(in: self.whitespaceAndPunctuation) + .lowercased() + } + + static func startsWithTrigger(transcript: String, triggers: [String]) -> Bool { + let tokens = transcript + .split(whereSeparator: { $0.isWhitespace }) + .map { self.normalizeToken(String($0)) } + .filter { !$0.isEmpty } + guard !tokens.isEmpty else { return false } + for trigger in triggers { + let triggerTokens = trigger + .split(whereSeparator: { $0.isWhitespace }) + .map { self.normalizeToken(String($0)) } + .filter { !$0.isEmpty } + guard !triggerTokens.isEmpty, tokens.count >= triggerTokens.count else { continue } + if zip(triggerTokens, tokens.prefix(triggerTokens.count)).allSatisfy({ $0 == $1 }) { + return true + } + } + return false + } + + static func textOnlyCommand( + transcript: String, + triggers: [String], + minCommandLength: Int, + trimWake: TrimWake) -> String? + { + guard !transcript.isEmpty else { return nil } + guard !self.normalizeToken(transcript).isEmpty else { return nil } + guard WakeWordGate.matchesTextOnly(text: transcript, triggers: triggers) else { return nil } + guard self.startsWithTrigger(transcript: transcript, triggers: triggers) else { return nil } + let trimmed = trimWake(transcript, triggers) + guard trimmed.count >= minCommandLength else { return nil } + return trimmed + } +} diff --git a/apps/macos/Sources/OpenClaw/WebChatManager.swift b/apps/macos/Sources/OpenClaw/WebChatManager.swift new file mode 100644 index 0000000000000000000000000000000000000000..2f77692de820d8a3fa5c3db2ce20466a34e0e95a --- /dev/null +++ b/apps/macos/Sources/OpenClaw/WebChatManager.swift @@ -0,0 +1,122 @@ +import AppKit +import Foundation + +/// A borderless panel that can still accept key focus (needed for typing). +final class WebChatPanel: NSPanel { + override var canBecomeKey: Bool { true } + override var canBecomeMain: Bool { true } +} + +enum WebChatPresentation { + case window + case panel(anchorProvider: () -> NSRect?) + + var isPanel: Bool { + if case .panel = self { return true } + return false + } +} + +@MainActor +final class WebChatManager { + static let shared = WebChatManager() + + private var windowController: WebChatSwiftUIWindowController? + private var windowSessionKey: String? + private var panelController: WebChatSwiftUIWindowController? + private var panelSessionKey: String? + private var cachedPreferredSessionKey: String? + + var onPanelVisibilityChanged: ((Bool) -> Void)? + + var activeSessionKey: String? { + self.panelSessionKey ?? self.windowSessionKey + } + + func show(sessionKey: String) { + self.closePanel() + if let controller = self.windowController { + if self.windowSessionKey == sessionKey { + controller.show() + return + } + + controller.close() + self.windowController = nil + self.windowSessionKey = nil + } + let controller = WebChatSwiftUIWindowController(sessionKey: sessionKey, presentation: .window) + controller.onVisibilityChanged = { [weak self] visible in + self?.onPanelVisibilityChanged?(visible) + } + self.windowController = controller + self.windowSessionKey = sessionKey + controller.show() + } + + func togglePanel(sessionKey: String, anchorProvider: @escaping () -> NSRect?) { + if let controller = self.panelController { + if self.panelSessionKey != sessionKey { + controller.close() + self.panelController = nil + self.panelSessionKey = nil + } else { + if controller.isVisible { + controller.close() + } else { + controller.presentAnchored(anchorProvider: anchorProvider) + } + return + } + } + + let controller = WebChatSwiftUIWindowController( + sessionKey: sessionKey, + presentation: .panel(anchorProvider: anchorProvider)) + controller.onClosed = { [weak self] in + self?.panelHidden() + } + controller.onVisibilityChanged = { [weak self] visible in + self?.onPanelVisibilityChanged?(visible) + } + self.panelController = controller + self.panelSessionKey = sessionKey + controller.presentAnchored(anchorProvider: anchorProvider) + } + + func closePanel() { + self.panelController?.close() + } + + func preferredSessionKey() async -> String { + if let cachedPreferredSessionKey { return cachedPreferredSessionKey } + let key = await GatewayConnection.shared.mainSessionKey() + self.cachedPreferredSessionKey = key + return key + } + + func resetTunnels() { + self.windowController?.close() + self.windowController = nil + self.windowSessionKey = nil + self.panelController?.close() + self.panelController = nil + self.panelSessionKey = nil + self.cachedPreferredSessionKey = nil + } + + func close() { + self.windowController?.close() + self.windowController = nil + self.windowSessionKey = nil + self.panelController?.close() + self.panelController = nil + self.panelSessionKey = nil + self.cachedPreferredSessionKey = nil + } + + private func panelHidden() { + self.onPanelVisibilityChanged?(false) + // Keep panel controller cached so reopening doesn't re-bootstrap. + } +} diff --git a/apps/macos/Sources/OpenClaw/WebChatSwiftUI.swift b/apps/macos/Sources/OpenClaw/WebChatSwiftUI.swift new file mode 100644 index 0000000000000000000000000000000000000000..d6b4417f06af3c5d26a07ad04cb17d630659e154 --- /dev/null +++ b/apps/macos/Sources/OpenClaw/WebChatSwiftUI.swift @@ -0,0 +1,374 @@ +import AppKit +import OpenClawChatUI +import OpenClawKit +import OpenClawProtocol +import Foundation +import OSLog +import QuartzCore +import SwiftUI + +private let webChatSwiftLogger = Logger(subsystem: "ai.openclaw", category: "WebChatSwiftUI") + +private enum WebChatSwiftUILayout { + static let windowSize = NSSize(width: 500, height: 840) + static let panelSize = NSSize(width: 480, height: 640) + static let windowMinSize = NSSize(width: 480, height: 360) + static let anchorPadding: CGFloat = 8 +} + +struct MacGatewayChatTransport: OpenClawChatTransport, Sendable { + func requestHistory(sessionKey: String) async throws -> OpenClawChatHistoryPayload { + try await GatewayConnection.shared.chatHistory(sessionKey: sessionKey) + } + + func abortRun(sessionKey: String, runId: String) async throws { + _ = try await GatewayConnection.shared.request( + method: "chat.abort", + params: [ + "sessionKey": AnyCodable(sessionKey), + "runId": AnyCodable(runId), + ], + timeoutMs: 10000) + } + + func listSessions(limit: Int?) async throws -> OpenClawChatSessionsListResponse { + var params: [String: AnyCodable] = [ + "includeGlobal": AnyCodable(true), + "includeUnknown": AnyCodable(false), + ] + if let limit { + params["limit"] = AnyCodable(limit) + } + let data = try await GatewayConnection.shared.request( + method: "sessions.list", + params: params, + timeoutMs: 15000) + return try JSONDecoder().decode(OpenClawChatSessionsListResponse.self, from: data) + } + + func sendMessage( + sessionKey: String, + message: String, + thinking: String, + idempotencyKey: String, + attachments: [OpenClawChatAttachmentPayload]) async throws -> OpenClawChatSendResponse + { + try await GatewayConnection.shared.chatSend( + sessionKey: sessionKey, + message: message, + thinking: thinking, + idempotencyKey: idempotencyKey, + attachments: attachments) + } + + func requestHealth(timeoutMs: Int) async throws -> Bool { + try await GatewayConnection.shared.healthOK(timeoutMs: timeoutMs) + } + + func events() -> AsyncStream { + AsyncStream { continuation in + let task = Task { + do { + try await GatewayConnection.shared.refresh() + } catch { + webChatSwiftLogger.error("gateway refresh failed \(error.localizedDescription, privacy: .public)") + } + + let stream = await GatewayConnection.shared.subscribe() + for await push in stream { + if Task.isCancelled { return } + if let evt = Self.mapPushToTransportEvent(push) { + continuation.yield(evt) + } + } + } + + continuation.onTermination = { @Sendable _ in + task.cancel() + } + } + } + + static func mapPushToTransportEvent(_ push: GatewayPush) -> OpenClawChatTransportEvent? { + switch push { + case let .snapshot(hello): + let ok = (try? JSONDecoder().decode( + OpenClawGatewayHealthOK.self, + from: JSONEncoder().encode(hello.snapshot.health)))?.ok ?? true + return .health(ok: ok) + + case let .event(evt): + switch evt.event { + case "health": + guard let payload = evt.payload else { return nil } + let ok = (try? JSONDecoder().decode( + OpenClawGatewayHealthOK.self, + from: JSONEncoder().encode(payload)))?.ok ?? true + return .health(ok: ok) + case "tick": + return .tick + case "chat": + guard let payload = evt.payload else { return nil } + guard let chat = try? JSONDecoder().decode( + OpenClawChatEventPayload.self, + from: JSONEncoder().encode(payload)) + else { + return nil + } + return .chat(chat) + case "agent": + guard let payload = evt.payload else { return nil } + guard let agent = try? JSONDecoder().decode( + OpenClawAgentEventPayload.self, + from: JSONEncoder().encode(payload)) + else { + return nil + } + return .agent(agent) + default: + return nil + } + + case .seqGap: + return .seqGap + } + } +} + +// MARK: - Window controller + +@MainActor +final class WebChatSwiftUIWindowController { + private let presentation: WebChatPresentation + private let sessionKey: String + private let hosting: NSHostingController + private let contentController: NSViewController + private var window: NSWindow? + private var dismissMonitor: Any? + var onClosed: (() -> Void)? + var onVisibilityChanged: ((Bool) -> Void)? + + convenience init(sessionKey: String, presentation: WebChatPresentation) { + self.init(sessionKey: sessionKey, presentation: presentation, transport: MacGatewayChatTransport()) + } + + init(sessionKey: String, presentation: WebChatPresentation, transport: any OpenClawChatTransport) { + self.sessionKey = sessionKey + self.presentation = presentation + let vm = OpenClawChatViewModel(sessionKey: sessionKey, transport: transport) + let accent = Self.color(fromHex: AppStateStore.shared.seamColorHex) + self.hosting = NSHostingController(rootView: OpenClawChatView( + viewModel: vm, + showsSessionSwitcher: true, + userAccent: accent)) + self.contentController = Self.makeContentController(for: presentation, hosting: self.hosting) + self.window = Self.makeWindow(for: presentation, contentViewController: self.contentController) + } + + deinit {} + + var isVisible: Bool { + self.window?.isVisible ?? false + } + + func show() { + guard let window else { return } + self.ensureWindowSize() + window.makeKeyAndOrderFront(nil) + NSApp.activate(ignoringOtherApps: true) + self.onVisibilityChanged?(true) + } + + func presentAnchored(anchorProvider: () -> NSRect?) { + guard case .panel = self.presentation, let window else { return } + self.installDismissMonitor() + let target = self.reposition(using: anchorProvider) + + if !self.isVisible { + let start = target.offsetBy(dx: 0, dy: 8) + window.setFrame(start, display: true) + window.alphaValue = 0 + window.makeKeyAndOrderFront(nil) + NSApp.activate(ignoringOtherApps: true) + NSAnimationContext.runAnimationGroup { context in + context.duration = 0.18 + context.timingFunction = CAMediaTimingFunction(name: .easeOut) + window.animator().setFrame(target, display: true) + window.animator().alphaValue = 1 + } + } else { + window.makeKeyAndOrderFront(nil) + NSApp.activate(ignoringOtherApps: true) + } + + self.onVisibilityChanged?(true) + } + + func close() { + self.window?.orderOut(nil) + self.onVisibilityChanged?(false) + self.onClosed?() + self.removeDismissMonitor() + } + + @discardableResult + private func reposition(using anchorProvider: () -> NSRect?) -> NSRect { + guard let window else { return .zero } + guard let anchor = anchorProvider() else { + let frame = WindowPlacement.topRightFrame( + size: WebChatSwiftUILayout.panelSize, + padding: WebChatSwiftUILayout.anchorPadding) + window.setFrame(frame, display: false) + return frame + } + let screen = NSScreen.screens.first { screen in + screen.frame.contains(anchor.origin) || screen.frame.contains(NSPoint(x: anchor.midX, y: anchor.midY)) + } ?? NSScreen.main + let bounds = (screen?.visibleFrame ?? .zero).insetBy( + dx: WebChatSwiftUILayout.anchorPadding, + dy: WebChatSwiftUILayout.anchorPadding) + let frame = WindowPlacement.anchoredBelowFrame( + size: WebChatSwiftUILayout.panelSize, + anchor: anchor, + padding: WebChatSwiftUILayout.anchorPadding, + in: bounds) + window.setFrame(frame, display: false) + return frame + } + + private func installDismissMonitor() { + if ProcessInfo.processInfo.isRunningTests { return } + guard self.dismissMonitor == nil, self.window != nil else { return } + self.dismissMonitor = NSEvent.addGlobalMonitorForEvents( + matching: [.leftMouseDown, .rightMouseDown, .otherMouseDown]) + { [weak self] _ in + guard let self, let win = self.window else { return } + let pt = NSEvent.mouseLocation + if !win.frame.contains(pt) { + self.close() + } + } + } + + private func removeDismissMonitor() { + if let monitor = self.dismissMonitor { + NSEvent.removeMonitor(monitor) + self.dismissMonitor = nil + } + } + + private static func makeWindow( + for presentation: WebChatPresentation, + contentViewController: NSViewController) -> NSWindow + { + switch presentation { + case .window: + let window = NSWindow( + contentRect: NSRect(origin: .zero, size: WebChatSwiftUILayout.windowSize), + styleMask: [.titled, .closable, .resizable, .miniaturizable], + backing: .buffered, + defer: false) + window.title = "OpenClaw Chat" + window.contentViewController = contentViewController + window.isReleasedWhenClosed = false + window.titleVisibility = .visible + window.titlebarAppearsTransparent = false + window.backgroundColor = .clear + window.isOpaque = false + window.center() + WindowPlacement.ensureOnScreen(window: window, defaultSize: WebChatSwiftUILayout.windowSize) + window.minSize = WebChatSwiftUILayout.windowMinSize + window.contentView?.wantsLayer = true + window.contentView?.layer?.backgroundColor = NSColor.clear.cgColor + return window + case .panel: + let panel = WebChatPanel( + contentRect: NSRect(origin: .zero, size: WebChatSwiftUILayout.panelSize), + styleMask: [.borderless], + backing: .buffered, + defer: false) + panel.level = .statusBar + panel.hidesOnDeactivate = true + panel.hasShadow = true + panel.isMovable = false + panel.collectionBehavior = [.canJoinAllSpaces, .fullScreenAuxiliary] + panel.titleVisibility = .hidden + panel.titlebarAppearsTransparent = true + panel.backgroundColor = .clear + panel.isOpaque = false + panel.contentViewController = contentViewController + panel.becomesKeyOnlyIfNeeded = true + panel.contentView?.wantsLayer = true + panel.contentView?.layer?.backgroundColor = NSColor.clear.cgColor + panel.setFrame( + WindowPlacement.topRightFrame( + size: WebChatSwiftUILayout.panelSize, + padding: WebChatSwiftUILayout.anchorPadding), + display: false) + return panel + } + } + + private static func makeContentController( + for presentation: WebChatPresentation, + hosting: NSHostingController) -> NSViewController + { + let controller = NSViewController() + let effectView = NSVisualEffectView() + effectView.material = .sidebar + effectView.blendingMode = .behindWindow + effectView.state = .active + effectView.wantsLayer = true + effectView.layer?.cornerCurve = .continuous + let cornerRadius: CGFloat = switch presentation { + case .panel: + 16 + case .window: + 0 + } + effectView.layer?.cornerRadius = cornerRadius + effectView.layer?.masksToBounds = true + + effectView.translatesAutoresizingMaskIntoConstraints = true + effectView.autoresizingMask = [.width, .height] + let rootView = effectView + + hosting.view.translatesAutoresizingMaskIntoConstraints = false + hosting.view.wantsLayer = true + hosting.view.layer?.backgroundColor = NSColor.clear.cgColor + + controller.addChild(hosting) + effectView.addSubview(hosting.view) + controller.view = rootView + + NSLayoutConstraint.activate([ + hosting.view.leadingAnchor.constraint(equalTo: effectView.leadingAnchor), + hosting.view.trailingAnchor.constraint(equalTo: effectView.trailingAnchor), + hosting.view.topAnchor.constraint(equalTo: effectView.topAnchor), + hosting.view.bottomAnchor.constraint(equalTo: effectView.bottomAnchor), + ]) + + return controller + } + + private func ensureWindowSize() { + guard case .window = self.presentation, let window else { return } + let current = window.frame.size + let min = WebChatSwiftUILayout.windowMinSize + if current.width < min.width || current.height < min.height { + let frame = WindowPlacement.centeredFrame(size: WebChatSwiftUILayout.windowSize) + window.setFrame(frame, display: false) + } + } + + private static func color(fromHex raw: String?) -> Color? { + let trimmed = (raw ?? "").trimmingCharacters(in: .whitespacesAndNewlines) + guard !trimmed.isEmpty else { return nil } + let hex = trimmed.hasPrefix("#") ? String(trimmed.dropFirst()) : trimmed + guard hex.count == 6, let value = Int(hex, radix: 16) else { return nil } + let r = Double((value >> 16) & 0xFF) / 255.0 + let g = Double((value >> 8) & 0xFF) / 255.0 + let b = Double(value & 0xFF) / 255.0 + return Color(red: r, green: g, blue: b) + } +} diff --git a/apps/macos/Sources/OpenClaw/WindowPlacement.swift b/apps/macos/Sources/OpenClaw/WindowPlacement.swift new file mode 100644 index 0000000000000000000000000000000000000000..a088dd743b36e76a261336f30dffce5d7c9bb1f6 --- /dev/null +++ b/apps/macos/Sources/OpenClaw/WindowPlacement.swift @@ -0,0 +1,84 @@ +import AppKit + +@MainActor +enum WindowPlacement { + static func centeredFrame(size: NSSize, on screen: NSScreen? = NSScreen.main) -> NSRect { + let bounds = (screen?.visibleFrame ?? NSScreen.screens.first?.visibleFrame ?? .zero) + return self.centeredFrame(size: size, in: bounds) + } + + static func topRightFrame( + size: NSSize, + padding: CGFloat, + on screen: NSScreen? = NSScreen.main) -> NSRect + { + let bounds = (screen?.visibleFrame ?? NSScreen.screens.first?.visibleFrame ?? .zero) + return self.topRightFrame(size: size, padding: padding, in: bounds) + } + + static func centeredFrame(size: NSSize, in bounds: NSRect) -> NSRect { + if bounds == .zero { + return NSRect(origin: .zero, size: size) + } + + let clampedWidth = min(size.width, bounds.width) + let clampedHeight = min(size.height, bounds.height) + + let x = round(bounds.minX + (bounds.width - clampedWidth) / 2) + let y = round(bounds.minY + (bounds.height - clampedHeight) / 2) + return NSRect(x: x, y: y, width: clampedWidth, height: clampedHeight) + } + + static func topRightFrame(size: NSSize, padding: CGFloat, in bounds: NSRect) -> NSRect { + if bounds == .zero { + return NSRect(origin: .zero, size: size) + } + + let clampedWidth = min(size.width, bounds.width) + let clampedHeight = min(size.height, bounds.height) + + let x = round(bounds.maxX - clampedWidth - padding) + let y = round(bounds.maxY - clampedHeight - padding) + return NSRect(x: x, y: y, width: clampedWidth, height: clampedHeight) + } + + static func anchoredBelowFrame(size: NSSize, anchor: NSRect, padding: CGFloat, in bounds: NSRect) -> NSRect { + if bounds == .zero { + let x = round(anchor.midX - size.width / 2) + let y = round(anchor.minY - size.height - padding) + return NSRect(x: x, y: y, width: size.width, height: size.height) + } + + let clampedWidth = min(size.width, bounds.width) + let clampedHeight = min(size.height, bounds.height) + + let desiredX = round(anchor.midX - clampedWidth / 2) + let desiredY = round(anchor.minY - clampedHeight - padding) + + let maxX = bounds.maxX - clampedWidth + let maxY = bounds.maxY - clampedHeight + + let x = maxX >= bounds.minX ? min(max(desiredX, bounds.minX), maxX) : bounds.minX + let y = maxY >= bounds.minY ? min(max(desiredY, bounds.minY), maxY) : bounds.minY + + return NSRect(x: x, y: y, width: clampedWidth, height: clampedHeight) + } + + static func ensureOnScreen( + window: NSWindow, + defaultSize: NSSize, + fallback: ((NSScreen?) -> NSRect)? = nil) + { + let frame = window.frame + let targetScreens = NSScreen.screens.isEmpty ? [NSScreen.main].compactMap(\.self) : NSScreen.screens + let isVisibleSomewhere = targetScreens.contains { screen in + frame.intersects(screen.visibleFrame.insetBy(dx: 12, dy: 12)) + } + + if isVisibleSomewhere { return } + + let screen = NSScreen.main ?? targetScreens.first + let next = fallback?(screen) ?? self.centeredFrame(size: defaultSize, on: screen) + window.setFrame(next, display: false) + } +} diff --git a/apps/macos/Sources/OpenClaw/WorkActivityStore.swift b/apps/macos/Sources/OpenClaw/WorkActivityStore.swift new file mode 100644 index 0000000000000000000000000000000000000000..b6fd97477fc24bf6e14ff8483344c04aa35afd2e --- /dev/null +++ b/apps/macos/Sources/OpenClaw/WorkActivityStore.swift @@ -0,0 +1,260 @@ +import OpenClawKit +import OpenClawProtocol +import Foundation +import Observation +import SwiftUI + +@MainActor +@Observable +final class WorkActivityStore { + static let shared = WorkActivityStore() + + struct Activity: Equatable { + let sessionKey: String + let role: SessionRole + let kind: ActivityKind + let label: String + let startedAt: Date + var lastUpdate: Date + } + + private(set) var current: Activity? + private(set) var iconState: IconState = .idle + private(set) var lastToolLabel: String? + private(set) var lastToolUpdatedAt: Date? + + private var jobs: [String: Activity] = [:] + private var tools: [String: Activity] = [:] + private var currentSessionKey: String? + private var toolSeqBySession: [String: Int] = [:] + + private var mainSessionKeyStorage = "main" + private let toolResultGrace: TimeInterval = 2.0 + + var mainSessionKey: String { self.mainSessionKeyStorage } + + func handleJob(sessionKey: String, state: String) { + let isStart = state.lowercased() == "started" || state.lowercased() == "streaming" + if isStart { + let activity = Activity( + sessionKey: sessionKey, + role: self.role(for: sessionKey), + kind: .job, + label: "job", + startedAt: Date(), + lastUpdate: Date()) + self.setJobActive(activity) + } else { + // Job ended (done/error/aborted/etc). Clear everything for this session. + self.clearTool(sessionKey: sessionKey) + self.clearJob(sessionKey: sessionKey) + } + } + + func handleTool( + sessionKey: String, + phase: String, + name: String?, + meta: String?, + args: [String: OpenClawProtocol.AnyCodable]?) + { + let toolKind = Self.mapToolKind(name) + let label = Self.buildLabel(name: name, meta: meta, args: args) + if phase.lowercased() == "start" { + self.lastToolLabel = label + self.lastToolUpdatedAt = Date() + self.toolSeqBySession[sessionKey, default: 0] += 1 + let activity = Activity( + sessionKey: sessionKey, + role: self.role(for: sessionKey), + kind: .tool(toolKind), + label: label, + startedAt: Date(), + lastUpdate: Date()) + self.setToolActive(activity) + } else { + // Delay removal slightly to avoid flicker on rapid result/start bursts. + let key = sessionKey + let seq = self.toolSeqBySession[key, default: 0] + Task { [weak self] in + let nsDelay = UInt64((self?.toolResultGrace ?? 0) * 1_000_000_000) + try? await Task.sleep(nanoseconds: nsDelay) + await MainActor.run { + guard let self else { return } + guard self.toolSeqBySession[key, default: 0] == seq else { return } + self.lastToolUpdatedAt = Date() + self.clearTool(sessionKey: key) + } + } + } + } + + func resolveIconState(override selection: IconOverrideSelection) { + switch selection { + case .system: + self.iconState = self.deriveIconState() + case .idle: + self.iconState = .idle + default: + let base = selection.toIconState() + switch base { + case let .workingMain(kind), + let .workingOther(kind): + self.iconState = .overridden(kind) + case let .overridden(kind): + self.iconState = .overridden(kind) + case .idle: + self.iconState = .idle + } + } + } + + private func setJobActive(_ activity: Activity) { + self.jobs[activity.sessionKey] = activity + // Main session preempts immediately. + if activity.role == .main { + self.currentSessionKey = activity.sessionKey + } else if self.currentSessionKey == nil || !self.isActive(sessionKey: self.currentSessionKey!) { + self.currentSessionKey = activity.sessionKey + } + self.refreshDerivedState() + } + + private func setToolActive(_ activity: Activity) { + self.tools[activity.sessionKey] = activity + // Main session preempts immediately. + if activity.role == .main { + self.currentSessionKey = activity.sessionKey + } else if self.currentSessionKey == nil || !self.isActive(sessionKey: self.currentSessionKey!) { + self.currentSessionKey = activity.sessionKey + } + self.refreshDerivedState() + } + + func setMainSessionKey(_ sessionKey: String) { + let trimmed = sessionKey.trimmingCharacters(in: .whitespacesAndNewlines) + guard !trimmed.isEmpty else { return } + guard trimmed != self.mainSessionKeyStorage else { return } + self.mainSessionKeyStorage = trimmed + if let current = self.currentSessionKey, !self.isActive(sessionKey: current) { + self.pickNextSession() + } + self.refreshDerivedState() + } + + private func clearJob(sessionKey: String) { + guard self.jobs[sessionKey] != nil else { return } + self.jobs.removeValue(forKey: sessionKey) + + if self.currentSessionKey == sessionKey, !self.isActive(sessionKey: sessionKey) { + self.pickNextSession() + } + self.refreshDerivedState() + } + + private func clearTool(sessionKey: String) { + guard self.tools[sessionKey] != nil else { return } + self.tools.removeValue(forKey: sessionKey) + + if self.currentSessionKey == sessionKey, !self.isActive(sessionKey: sessionKey) { + self.pickNextSession() + } + self.refreshDerivedState() + } + + private func pickNextSession() { + // Prefer main if present. + if self.isActive(sessionKey: self.mainSessionKeyStorage) { + self.currentSessionKey = self.mainSessionKeyStorage + return + } + + // Otherwise, pick most recent by lastUpdate across job/tool. + let keys = Set(self.jobs.keys).union(self.tools.keys) + let next = keys.max(by: { self.lastUpdate(for: $0) < self.lastUpdate(for: $1) }) + self.currentSessionKey = next + } + + private func role(for sessionKey: String) -> SessionRole { + sessionKey == self.mainSessionKeyStorage ? .main : .other + } + + private func isActive(sessionKey: String) -> Bool { + self.jobs[sessionKey] != nil || self.tools[sessionKey] != nil + } + + private func lastUpdate(for sessionKey: String) -> Date { + max(self.jobs[sessionKey]?.lastUpdate ?? .distantPast, self.tools[sessionKey]?.lastUpdate ?? .distantPast) + } + + private func currentActivity(for sessionKey: String) -> Activity? { + // Prefer tool overlay if present, otherwise job. + self.tools[sessionKey] ?? self.jobs[sessionKey] + } + + private func refreshDerivedState() { + if let key = self.currentSessionKey, !self.isActive(sessionKey: key) { + self.currentSessionKey = nil + } + self.current = self.currentSessionKey.flatMap { self.currentActivity(for: $0) } + self.iconState = self.deriveIconState() + } + + private func deriveIconState() -> IconState { + guard let sessionKey = self.currentSessionKey, + let activity = self.currentActivity(for: sessionKey) + else { return .idle } + + switch activity.role { + case .main: return .workingMain(activity.kind) + case .other: return .workingOther(activity.kind) + } + } + + private static func mapToolKind(_ name: String?) -> ToolKind { + switch name?.lowercased() { + case "bash", "shell": .bash + case "read": .read + case "write": .write + case "edit": .edit + case "attach": .attach + default: .other + } + } + + private static func buildLabel( + name: String?, + meta: String?, + args: [String: OpenClawProtocol.AnyCodable]?) -> String + { + let wrappedArgs = self.wrapToolArgs(args) + let display = ToolDisplayRegistry.resolve(name: name ?? "tool", args: wrappedArgs, meta: meta) + if let detail = display.detailLine, !detail.isEmpty { + return "\(display.label): \(detail)" + } + + return display.label + } + + private static func wrapToolArgs(_ args: [String: OpenClawProtocol.AnyCodable]?) -> OpenClawKit.AnyCodable? { + guard let args else { return nil } + let converted: [String: Any] = args.mapValues { self.unwrapJSONValue($0.value) } + return OpenClawKit.AnyCodable(converted) + } + + private static func unwrapJSONValue(_ value: Any) -> Any { + if let dict = value as? [String: OpenClawProtocol.AnyCodable] { + return dict.mapValues { self.unwrapJSONValue($0.value) } + } + if let array = value as? [OpenClawProtocol.AnyCodable] { + return array.map { self.unwrapJSONValue($0.value) } + } + if let dict = value as? [String: Any] { + return dict.mapValues { self.unwrapJSONValue($0) } + } + if let array = value as? [Any] { + return array.map { self.unwrapJSONValue($0) } + } + return value + } +} diff --git a/apps/macos/Sources/OpenClawDiscovery/GatewayDiscoveryModel.swift b/apps/macos/Sources/OpenClawDiscovery/GatewayDiscoveryModel.swift new file mode 100644 index 0000000000000000000000000000000000000000..c8cde804ece5067e47d1fe13ef9a3bb66ae193bc --- /dev/null +++ b/apps/macos/Sources/OpenClawDiscovery/GatewayDiscoveryModel.swift @@ -0,0 +1,684 @@ +import OpenClawKit +import Foundation +import Network +import Observation +import OSLog + +@MainActor +@Observable +public final class GatewayDiscoveryModel { + public struct LocalIdentity: Equatable, Sendable { + public var hostTokens: Set + public var displayTokens: Set + + public init(hostTokens: Set, displayTokens: Set) { + self.hostTokens = hostTokens + self.displayTokens = displayTokens + } + } + + public struct DiscoveredGateway: Identifiable, Equatable, Sendable { + public var id: String { self.stableID } + public var displayName: String + public var lanHost: String? + public var tailnetDns: String? + public var sshPort: Int + public var gatewayPort: Int? + public var cliPath: String? + public var stableID: String + public var debugID: String + public var isLocal: Bool + + public init( + displayName: String, + lanHost: String? = nil, + tailnetDns: String? = nil, + sshPort: Int, + gatewayPort: Int? = nil, + cliPath: String? = nil, + stableID: String, + debugID: String, + isLocal: Bool) + { + self.displayName = displayName + self.lanHost = lanHost + self.tailnetDns = tailnetDns + self.sshPort = sshPort + self.gatewayPort = gatewayPort + self.cliPath = cliPath + self.stableID = stableID + self.debugID = debugID + self.isLocal = isLocal + } + } + + public var gateways: [DiscoveredGateway] = [] + public var statusText: String = "Idle" + + private var browsers: [String: NWBrowser] = [:] + private var resultsByDomain: [String: Set] = [:] + private var gatewaysByDomain: [String: [DiscoveredGateway]] = [:] + private var statesByDomain: [String: NWBrowser.State] = [:] + private var localIdentity: LocalIdentity + private let localDisplayName: String? + private let filterLocalGateways: Bool + private var resolvedTXTByID: [String: [String: String]] = [:] + private var pendingTXTResolvers: [String: GatewayTXTResolver] = [:] + private var wideAreaFallbackTask: Task? + private var wideAreaFallbackGateways: [DiscoveredGateway] = [] + private let logger = Logger(subsystem: "ai.openclaw", category: "gateway-discovery") + + public init( + localDisplayName: String? = nil, + filterLocalGateways: Bool = true) + { + self.localDisplayName = localDisplayName + self.filterLocalGateways = filterLocalGateways + self.localIdentity = Self.buildLocalIdentityFast(displayName: localDisplayName) + self.refreshLocalIdentity() + } + + public func start() { + if !self.browsers.isEmpty { return } + + for domain in OpenClawBonjour.gatewayServiceDomains { + let params = NWParameters.tcp + params.includePeerToPeer = true + let browser = NWBrowser( + for: .bonjour(type: OpenClawBonjour.gatewayServiceType, domain: domain), + using: params) + + browser.stateUpdateHandler = { [weak self] state in + Task { @MainActor in + guard let self else { return } + self.statesByDomain[domain] = state + self.updateStatusText() + } + } + + browser.browseResultsChangedHandler = { [weak self] results, _ in + Task { @MainActor in + guard let self else { return } + self.resultsByDomain[domain] = results + self.updateGateways(for: domain) + self.recomputeGateways() + } + } + + self.browsers[domain] = browser + browser.start(queue: DispatchQueue(label: "ai.openclaw.macos.gateway-discovery.\(domain)")) + } + + self.scheduleWideAreaFallback() + } + + public func refreshWideAreaFallbackNow(timeoutSeconds: TimeInterval = 5.0) { + guard let domain = OpenClawBonjour.wideAreaGatewayServiceDomain else { return } + Task.detached(priority: .utility) { [weak self] in + guard let self else { return } + let beacons = WideAreaGatewayDiscovery.discover(timeoutSeconds: timeoutSeconds) + await MainActor.run { [weak self] in + guard let self else { return } + self.wideAreaFallbackGateways = self.mapWideAreaBeacons(beacons, domain: domain) + self.recomputeGateways() + } + } + } + + public func stop() { + for browser in self.browsers.values { + browser.cancel() + } + self.browsers = [:] + self.resultsByDomain = [:] + self.gatewaysByDomain = [:] + self.statesByDomain = [:] + self.resolvedTXTByID = [:] + self.pendingTXTResolvers.values.forEach { $0.cancel() } + self.pendingTXTResolvers = [:] + self.wideAreaFallbackTask?.cancel() + self.wideAreaFallbackTask = nil + self.wideAreaFallbackGateways = [] + self.gateways = [] + self.statusText = "Stopped" + } + + private func mapWideAreaBeacons(_ beacons: [WideAreaGatewayBeacon], domain: String) -> [DiscoveredGateway] { + beacons.map { beacon in + let stableID = "wide-area|\(domain)|\(beacon.instanceName)" + let isLocal = Self.isLocalGateway( + lanHost: beacon.lanHost, + tailnetDns: beacon.tailnetDns, + displayName: beacon.displayName, + serviceName: beacon.instanceName, + local: self.localIdentity) + return DiscoveredGateway( + displayName: beacon.displayName, + lanHost: beacon.lanHost, + tailnetDns: beacon.tailnetDns, + sshPort: beacon.sshPort ?? 22, + gatewayPort: beacon.gatewayPort, + cliPath: beacon.cliPath, + stableID: stableID, + debugID: "\(beacon.instanceName)@\(beacon.host):\(beacon.port)", + isLocal: isLocal) + } + } + + private func recomputeGateways() { + let primary = self.sortedDeduped(gateways: self.gatewaysByDomain.values.flatMap(\.self)) + let primaryFiltered = self.filterLocalGateways ? primary.filter { !$0.isLocal } : primary + if !primaryFiltered.isEmpty { + self.gateways = primaryFiltered + return + } + + // Bonjour can return only "local" results for the wide-area domain (or no results at all), + // which makes onboarding look empty even though Tailscale DNS-SD can already see gateways. + guard !self.wideAreaFallbackGateways.isEmpty else { + self.gateways = primaryFiltered + return + } + + let combined = self.sortedDeduped(gateways: primary + self.wideAreaFallbackGateways) + self.gateways = self.filterLocalGateways ? combined.filter { !$0.isLocal } : combined + } + + private func updateGateways(for domain: String) { + guard let results = self.resultsByDomain[domain] else { + self.gatewaysByDomain[domain] = [] + return + } + + self.gatewaysByDomain[domain] = results.compactMap { result -> DiscoveredGateway? in + guard case let .service(name, type, resultDomain, _) = result.endpoint else { return nil } + + let decodedName = BonjourEscapes.decode(name) + let stableID = GatewayEndpointID.stableID(result.endpoint) + let resolvedTXT = self.resolvedTXTByID[stableID] ?? [:] + let txt = Self.txtDictionary(from: result).merging( + resolvedTXT, + uniquingKeysWith: { _, new in new }) + + let advertisedName = txt["displayName"] + .map(Self.prettifyInstanceName) + .flatMap { $0.isEmpty ? nil : $0 } + let prettyName = + advertisedName ?? Self.prettifyServiceName(decodedName) + + let parsedTXT = Self.parseGatewayTXT(txt) + + if parsedTXT.lanHost == nil || parsedTXT.tailnetDns == nil { + self.ensureTXTResolution( + stableID: stableID, + serviceName: name, + type: type, + domain: resultDomain) + } + + let isLocal = Self.isLocalGateway( + lanHost: parsedTXT.lanHost, + tailnetDns: parsedTXT.tailnetDns, + displayName: prettyName, + serviceName: decodedName, + local: self.localIdentity) + return DiscoveredGateway( + displayName: prettyName, + lanHost: parsedTXT.lanHost, + tailnetDns: parsedTXT.tailnetDns, + sshPort: parsedTXT.sshPort, + gatewayPort: parsedTXT.gatewayPort, + cliPath: parsedTXT.cliPath, + stableID: stableID, + debugID: GatewayEndpointID.prettyDescription(result.endpoint), + isLocal: isLocal) + } + .sorted { $0.displayName.localizedCaseInsensitiveCompare($1.displayName) == .orderedAscending } + + if let wideAreaDomain = OpenClawBonjour.wideAreaGatewayServiceDomain, + domain == wideAreaDomain, + self.hasUsableWideAreaResults + { + self.wideAreaFallbackGateways = [] + } + } + + private func scheduleWideAreaFallback() { + guard let domain = OpenClawBonjour.wideAreaGatewayServiceDomain else { return } + if Self.isRunningTests { return } + guard self.wideAreaFallbackTask == nil else { return } + self.wideAreaFallbackTask = Task.detached(priority: .utility) { [weak self] in + guard let self else { return } + var attempt = 0 + let startedAt = Date() + while !Task.isCancelled, Date().timeIntervalSince(startedAt) < 35.0 { + let hasResults = await MainActor.run { + self.hasUsableWideAreaResults + } + if hasResults { return } + + // Wide-area discovery can be racy (Tailscale not yet up, DNS zone not + // published yet). Retry with a short backoff while onboarding is open. + let beacons = WideAreaGatewayDiscovery.discover(timeoutSeconds: 2.0) + if !beacons.isEmpty { + await MainActor.run { [weak self] in + guard let self else { return } + self.wideAreaFallbackGateways = self.mapWideAreaBeacons(beacons, domain: domain) + self.recomputeGateways() + } + return + } + + attempt += 1 + let backoff = min(8.0, 0.6 + (Double(attempt) * 0.7)) + try? await Task.sleep(nanoseconds: UInt64(backoff * 1_000_000_000)) + } + } + } + + private var hasUsableWideAreaResults: Bool { + guard let domain = OpenClawBonjour.wideAreaGatewayServiceDomain else { return false } + guard let gateways = self.gatewaysByDomain[domain], !gateways.isEmpty else { return false } + if !self.filterLocalGateways { return true } + return gateways.contains(where: { !$0.isLocal }) + } + + private func sortedDeduped(gateways: [DiscoveredGateway]) -> [DiscoveredGateway] { + var seen = Set() + let deduped = gateways.filter { gateway in + if seen.contains(gateway.stableID) { return false } + seen.insert(gateway.stableID) + return true + } + return deduped.sorted { + $0.displayName.localizedCaseInsensitiveCompare($1.displayName) == .orderedAscending + } + } + + private nonisolated static var isRunningTests: Bool { + // Keep discovery background work from running forever during SwiftPM test runs. + if Bundle.allBundles.contains(where: { $0.bundleURL.pathExtension == "xctest" }) { return true } + + let env = ProcessInfo.processInfo.environment + return env["XCTestConfigurationFilePath"] != nil + || env["XCTestBundlePath"] != nil + || env["XCTestSessionIdentifier"] != nil + } + + private func updateGatewaysForAllDomains() { + for domain in self.resultsByDomain.keys { + self.updateGateways(for: domain) + } + } + + private func updateStatusText() { + let states = Array(self.statesByDomain.values) + if states.isEmpty { + self.statusText = self.browsers.isEmpty ? "Idle" : "Setup" + return + } + + if let failed = states.first(where: { state in + if case .failed = state { return true } + return false + }) { + if case let .failed(err) = failed { + self.statusText = "Failed: \(err)" + return + } + } + + if let waiting = states.first(where: { state in + if case .waiting = state { return true } + return false + }) { + if case let .waiting(err) = waiting { + self.statusText = "Waiting: \(err)" + return + } + } + + if states.contains(where: { if case .ready = $0 { true } else { false } }) { + self.statusText = "Searching…" + return + } + + if states.contains(where: { if case .setup = $0 { true } else { false } }) { + self.statusText = "Setup" + return + } + + self.statusText = "Searching…" + } + + private static func txtDictionary(from result: NWBrowser.Result) -> [String: String] { + var merged: [String: String] = [:] + + if case let .bonjour(txt) = result.metadata { + merged.merge(txt.dictionary, uniquingKeysWith: { _, new in new }) + } + + if let endpointTxt = result.endpoint.txtRecord?.dictionary { + merged.merge(endpointTxt, uniquingKeysWith: { _, new in new }) + } + + return merged + } + + public struct GatewayTXT: Equatable { + public var lanHost: String? + public var tailnetDns: String? + public var sshPort: Int + public var gatewayPort: Int? + public var cliPath: String? + } + + public static func parseGatewayTXT(_ txt: [String: String]) -> GatewayTXT { + var lanHost: String? + var tailnetDns: String? + var sshPort = 22 + var gatewayPort: Int? + var cliPath: String? + + if let value = txt["lanHost"] { + let trimmed = value.trimmingCharacters(in: .whitespacesAndNewlines) + lanHost = trimmed.isEmpty ? nil : trimmed + } + if let value = txt["tailnetDns"] { + let trimmed = value.trimmingCharacters(in: .whitespacesAndNewlines) + tailnetDns = trimmed.isEmpty ? nil : trimmed + } + if let value = txt["sshPort"], + let parsed = Int(value.trimmingCharacters(in: .whitespacesAndNewlines)), + parsed > 0 + { + sshPort = parsed + } + if let value = txt["gatewayPort"], + let parsed = Int(value.trimmingCharacters(in: .whitespacesAndNewlines)), + parsed > 0 + { + gatewayPort = parsed + } + if let value = txt["cliPath"] { + let trimmed = value.trimmingCharacters(in: .whitespacesAndNewlines) + cliPath = trimmed.isEmpty ? nil : trimmed + } + + return GatewayTXT( + lanHost: lanHost, + tailnetDns: tailnetDns, + sshPort: sshPort, + gatewayPort: gatewayPort, + cliPath: cliPath) + } + + public static func buildSSHTarget(user: String, host: String, port: Int) -> String { + var target = "\(user)@\(host)" + if port != 22 { + target += ":\(port)" + } + return target + } + + private func ensureTXTResolution( + stableID: String, + serviceName: String, + type: String, + domain: String) + { + guard self.resolvedTXTByID[stableID] == nil else { return } + guard self.pendingTXTResolvers[stableID] == nil else { return } + + let resolver = GatewayTXTResolver( + name: serviceName, + type: type, + domain: domain, + logger: self.logger) + { [weak self] result in + Task { @MainActor in + guard let self else { return } + self.pendingTXTResolvers[stableID] = nil + switch result { + case let .success(txt): + self.resolvedTXTByID[stableID] = txt + self.updateGatewaysForAllDomains() + self.recomputeGateways() + case .failure: + break + } + } + } + + self.pendingTXTResolvers[stableID] = resolver + resolver.start() + } + + private nonisolated static func prettifyInstanceName(_ decodedName: String) -> String { + let normalized = decodedName.split(whereSeparator: \.isWhitespace).joined(separator: " ") + let stripped = normalized.replacingOccurrences(of: " (OpenClaw)", with: "") + .replacingOccurrences(of: #"\s+\(\d+\)$"#, with: "", options: .regularExpression) + return stripped.trimmingCharacters(in: .whitespacesAndNewlines) + } + + private nonisolated static func prettifyServiceName(_ decodedName: String) -> String { + let normalized = Self.prettifyInstanceName(decodedName) + var cleaned = normalized.replacingOccurrences(of: #"\s*-?gateway$"#, with: "", options: .regularExpression) + cleaned = cleaned + .replacingOccurrences(of: "_", with: " ") + .replacingOccurrences(of: "-", with: " ") + .replacingOccurrences(of: #"\s+"#, with: " ", options: .regularExpression) + .trimmingCharacters(in: .whitespacesAndNewlines) + if cleaned.isEmpty { + cleaned = normalized + } + let words = cleaned.split(separator: " ") + let titled = words.map { word -> String in + let lower = word.lowercased() + guard let first = lower.first else { return "" } + return String(first).uppercased() + lower.dropFirst() + }.joined(separator: " ") + return titled.isEmpty ? normalized : titled + } + + public nonisolated static func isLocalGateway( + lanHost: String?, + tailnetDns: String?, + displayName: String?, + serviceName: String?, + local: LocalIdentity) -> Bool + { + if let host = normalizeHostToken(lanHost), + local.hostTokens.contains(host) + { + return true + } + if let host = normalizeHostToken(tailnetDns), + local.hostTokens.contains(host) + { + return true + } + if let name = normalizeDisplayToken(displayName), + local.displayTokens.contains(name) + { + return true + } + if let serviceHost = normalizeServiceHostToken(serviceName), + local.hostTokens.contains(serviceHost) + { + return true + } + return false + } + + private func refreshLocalIdentity() { + let fastIdentity = self.localIdentity + let displayName = self.localDisplayName + Task.detached(priority: .utility) { + let slowIdentity = Self.buildLocalIdentitySlow(displayName: displayName) + let merged = Self.mergeLocalIdentity(fast: fastIdentity, slow: slowIdentity) + await MainActor.run { [weak self] in + guard let self else { return } + guard self.localIdentity != merged else { return } + self.localIdentity = merged + self.recomputeGateways() + } + } + } + + private nonisolated static func mergeLocalIdentity( + fast: LocalIdentity, + slow: LocalIdentity) -> LocalIdentity + { + LocalIdentity( + hostTokens: fast.hostTokens.union(slow.hostTokens), + displayTokens: fast.displayTokens.union(slow.displayTokens)) + } + + private nonisolated static func buildLocalIdentityFast(displayName: String?) -> LocalIdentity { + var hostTokens: Set = [] + var displayTokens: Set = [] + + let hostName = ProcessInfo.processInfo.hostName + if let token = normalizeHostToken(hostName) { + hostTokens.insert(token) + } + + if let token = normalizeDisplayToken(displayName) { + displayTokens.insert(token) + } + + return LocalIdentity(hostTokens: hostTokens, displayTokens: displayTokens) + } + + private nonisolated static func buildLocalIdentitySlow(displayName: String?) -> LocalIdentity { + var hostTokens: Set = [] + var displayTokens: Set = [] + + if let host = Host.current().name, + let token = normalizeHostToken(host) + { + hostTokens.insert(token) + } + + if let token = normalizeDisplayToken(displayName) { + displayTokens.insert(token) + } + + if let token = normalizeDisplayToken(Host.current().localizedName) { + displayTokens.insert(token) + } + + return LocalIdentity(hostTokens: hostTokens, displayTokens: displayTokens) + } + + private nonisolated static func normalizeHostToken(_ raw: String?) -> String? { + guard let raw else { return nil } + let trimmed = raw.trimmingCharacters(in: .whitespacesAndNewlines) + if trimmed.isEmpty { return nil } + let lower = trimmed.lowercased() + let strippedTrailingDot = lower.hasSuffix(".") + ? String(lower.dropLast()) + : lower + let withoutLocal = strippedTrailingDot.hasSuffix(".local") + ? String(strippedTrailingDot.dropLast(6)) + : strippedTrailingDot + let firstLabel = withoutLocal.split(separator: ".").first.map(String.init) + let token = (firstLabel ?? withoutLocal).trimmingCharacters(in: .whitespacesAndNewlines) + return token.isEmpty ? nil : token + } + + private nonisolated static func normalizeDisplayToken(_ raw: String?) -> String? { + guard let raw else { return nil } + let prettified = Self.prettifyInstanceName(raw) + let trimmed = prettified.trimmingCharacters(in: .whitespacesAndNewlines) + if trimmed.isEmpty { return nil } + return trimmed.lowercased() + } + + private nonisolated static func normalizeServiceHostToken(_ raw: String?) -> String? { + guard let raw else { return nil } + let prettified = Self.prettifyInstanceName(raw) + let strippedGateway = prettified.replacingOccurrences( + of: #"\s*-?\s*gateway$"#, + with: "", + options: .regularExpression) + return self.normalizeHostToken(strippedGateway) + } +} + +final class GatewayTXTResolver: NSObject, NetServiceDelegate { + private let service: NetService + private let completion: (Result<[String: String], Error>) -> Void + private let logger: Logger + private var didFinish = false + + init( + name: String, + type: String, + domain: String, + logger: Logger, + completion: @escaping (Result<[String: String], Error>) -> Void) + { + self.service = NetService(domain: domain, type: type, name: name) + self.completion = completion + self.logger = logger + super.init() + self.service.delegate = self + } + + func start(timeout: TimeInterval = 2.0) { + self.service.schedule(in: .main, forMode: .common) + self.service.resolve(withTimeout: timeout) + } + + func cancel() { + self.finish(result: .failure(GatewayTXTResolverError.cancelled)) + } + + func netServiceDidResolveAddress(_ sender: NetService) { + let txt = Self.decodeTXT(sender.txtRecordData()) + if !txt.isEmpty { + let payload = self.formatTXT(txt) + self.logger.debug( + "discovery: resolved TXT for \(sender.name, privacy: .public): \(payload, privacy: .public)") + } + self.finish(result: .success(txt)) + } + + func netService(_ sender: NetService, didNotResolve errorDict: [String: NSNumber]) { + self.finish(result: .failure(GatewayTXTResolverError.resolveFailed(errorDict))) + } + + private func finish(result: Result<[String: String], Error>) { + guard !self.didFinish else { return } + self.didFinish = true + self.service.stop() + self.service.remove(from: .main, forMode: .common) + self.completion(result) + } + + private static func decodeTXT(_ data: Data?) -> [String: String] { + guard let data else { return [:] } + let dict = NetService.dictionary(fromTXTRecord: data) + var out: [String: String] = [:] + out.reserveCapacity(dict.count) + for (key, value) in dict { + if let str = String(data: value, encoding: .utf8) { + out[key] = str + } + } + return out + } + + private func formatTXT(_ txt: [String: String]) -> String { + txt.sorted(by: { $0.key < $1.key }) + .map { "\($0.key)=\($0.value)" } + .joined(separator: " ") + } +} + +enum GatewayTXTResolverError: Error { + case cancelled + case resolveFailed([String: NSNumber]) +} diff --git a/apps/macos/Sources/OpenClawDiscovery/WideAreaGatewayDiscovery.swift b/apps/macos/Sources/OpenClawDiscovery/WideAreaGatewayDiscovery.swift new file mode 100644 index 0000000000000000000000000000000000000000..bacff45d604cbaf61bbc45f9cd331334f82ab3a9 --- /dev/null +++ b/apps/macos/Sources/OpenClawDiscovery/WideAreaGatewayDiscovery.swift @@ -0,0 +1,374 @@ +import OpenClawKit +import Foundation + +struct WideAreaGatewayBeacon: Sendable, Equatable { + var instanceName: String + var displayName: String + var host: String + var port: Int + var lanHost: String? + var tailnetDns: String? + var gatewayPort: Int? + var sshPort: Int? + var cliPath: String? +} + +enum WideAreaGatewayDiscovery { + private static let maxCandidates = 40 + private static let digPath = "/usr/bin/dig" + private static let defaultTimeoutSeconds: TimeInterval = 0.2 + private static let nameserverProbeConcurrency = 6 + + struct DiscoveryContext: Sendable { + var tailscaleStatus: @Sendable () -> String? + var dig: @Sendable (_ args: [String], _ timeout: TimeInterval) -> String? + + static let live = DiscoveryContext( + tailscaleStatus: { readTailscaleStatus() }, + dig: { args, timeout in + runDig(args: args, timeout: timeout) + }) + } + + static func discover( + timeoutSeconds: TimeInterval = 2.0, + context: DiscoveryContext = .live) -> [WideAreaGatewayBeacon] + { + let startedAt = Date() + let remaining = { + timeoutSeconds - Date().timeIntervalSince(startedAt) + } + + guard let ips = collectTailnetIPv4s( + statusJson: context.tailscaleStatus()).nonEmpty else { return [] } + var candidates = Array(ips.prefix(self.maxCandidates)) + guard let nameserver = findNameserver( + candidates: &candidates, + remaining: remaining, + dig: context.dig) + else { + return [] + } + + guard let domain = OpenClawBonjour.wideAreaGatewayServiceDomain else { return [] } + let domainTrimmed = domain.trimmingCharacters(in: CharacterSet(charactersIn: ".")) + let probeName = "_openclaw-gw._tcp.\(domainTrimmed)" + guard let ptrLines = context.dig( + ["+short", "+time=1", "+tries=1", "@\(nameserver)", probeName, "PTR"], + min(defaultTimeoutSeconds, remaining()))?.split(whereSeparator: \.isNewline), + !ptrLines.isEmpty + else { + return [] + } + + var beacons: [WideAreaGatewayBeacon] = [] + for raw in ptrLines { + let ptr = raw.trimmingCharacters(in: .whitespacesAndNewlines) + if ptr.isEmpty { continue } + let ptrName = ptr.hasSuffix(".") ? String(ptr.dropLast()) : ptr + let suffix = "._openclaw-gw._tcp.\(domainTrimmed)" + let rawInstanceName = ptrName.hasSuffix(suffix) + ? String(ptrName.dropLast(suffix.count)) + : ptrName + let instanceName = self.decodeDnsSdEscapes(rawInstanceName) + + guard let srv = context.dig( + ["+short", "+time=1", "+tries=1", "@\(nameserver)", ptrName, "SRV"], + min(defaultTimeoutSeconds, remaining())) + else { continue } + guard let (host, port) = parseSrv(srv) else { continue } + + let txtRaw = context.dig( + ["+short", "+time=1", "+tries=1", "@\(nameserver)", ptrName, "TXT"], + min(self.defaultTimeoutSeconds, remaining())) + let txtTokens = txtRaw.map(self.parseTxtTokens) ?? [] + let txt = self.mapTxt(tokens: txtTokens) + + let displayName = txt["displayName"] ?? instanceName + let beacon = WideAreaGatewayBeacon( + instanceName: instanceName, + displayName: displayName, + host: host, + port: port, + lanHost: txt["lanHost"], + tailnetDns: txt["tailnetDns"], + gatewayPort: parseInt(txt["gatewayPort"]), + sshPort: parseInt(txt["sshPort"]), + cliPath: txt["cliPath"]) + beacons.append(beacon) + } + + return beacons + } + + private static func collectTailnetIPv4s(statusJson: String?) -> [String] { + guard let statusJson else { return [] } + let decoder = JSONDecoder() + guard let data = statusJson.data(using: .utf8), + let status = try? decoder.decode(TailscaleStatus.self, from: data) + else { return [] } + + var ips: [String] = [] + ips.append(contentsOf: status.selfNode?.resolvedIPs ?? []) + if let peers = status.peer { + for peer in peers.values { + ips.append(contentsOf: peer.resolvedIPs) + } + } + + var seen = Set() + let ordered = ips.filter { value in + guard self.isTailnetIPv4(value) else { return false } + if seen.contains(value) { return false } + seen.insert(value) + return true + } + return ordered + } + + private static func readTailscaleStatus() -> String? { + let candidates = [ + "/usr/local/bin/tailscale", + "/opt/homebrew/bin/tailscale", + "/Applications/Tailscale.app/Contents/MacOS/Tailscale", + "tailscale", + ] + + var output: String? + for candidate in candidates { + if let result = run( + path: candidate, + args: ["status", "--json"], + timeout: 0.7) + { + output = result + break + } + } + + return output + } + + private static func findNameserver( + candidates: inout [String], + remaining: () -> TimeInterval, + dig: @escaping @Sendable (_ args: [String], _ timeout: TimeInterval) -> String?) -> String? + { + guard let domain = OpenClawBonjour.wideAreaGatewayServiceDomain else { return nil } + let domainTrimmed = domain.trimmingCharacters(in: CharacterSet(charactersIn: ".")) + let probeName = "_openclaw-gw._tcp.\(domainTrimmed)" + + let ips = candidates + candidates.removeAll(keepingCapacity: true) + if ips.isEmpty { return nil } + + final class ProbeState: @unchecked Sendable { + let lock = NSLock() + var nextIndex = 0 + var found: String? + } + + let state = ProbeState() + let deadline = Date().addingTimeInterval(max(0, remaining())) + let workerCount = min(self.nameserverProbeConcurrency, ips.count) + let group = DispatchGroup() + + for _ in 0..= ips.count { return } + let ip = ips[i] + let budget = deadline.timeIntervalSinceNow + if budget <= 0 { return } + + if let stdout = dig( + ["+short", "+time=1", "+tries=1", "@\(ip)", probeName, "PTR"], + min(defaultTimeoutSeconds, budget)), + stdout.split(whereSeparator: \.isNewline).isEmpty == false + { + state.lock.lock() + if state.found == nil { + state.found = ip + } + state.lock.unlock() + return + } + } + } + } + + _ = group.wait(timeout: .now() + max(0.0, remaining())) + return state.found + } + + private static func runDig(args: [String], timeout: TimeInterval) -> String? { + self.run(path: self.digPath, args: args, timeout: timeout) + } + + private static func run(path: String, args: [String], timeout: TimeInterval) -> String? { + let process = Process() + process.executableURL = URL(fileURLWithPath: path) + process.arguments = args + let outPipe = Pipe() + process.standardOutput = outPipe + // Avoid stderr pipe backpressure; we don't consume it. + process.standardError = FileHandle.nullDevice + + do { + try process.run() + } catch { + return nil + } + + let deadline = Date().addingTimeInterval(timeout) + while process.isRunning, Date() < deadline { + Thread.sleep(forTimeInterval: 0.02) + } + if process.isRunning { + process.terminate() + } + process.waitUntilExit() + + let data = (try? outPipe.fileHandleForReading.readToEnd()) ?? Data() + let output = String(data: data, encoding: .utf8)?.trimmingCharacters(in: .whitespacesAndNewlines) + return output?.isEmpty == false ? output : nil + } + + private static func parseSrv(_ stdout: String) -> (String, Int)? { + let line = stdout + .split(whereSeparator: \.isNewline) + .map { $0.trimmingCharacters(in: .whitespacesAndNewlines) } + .first(where: { !$0.isEmpty }) + guard let line else { return nil } + let parts = line.split(whereSeparator: { $0 == " " || $0 == "\t" }).map(String.init) + guard parts.count >= 4 else { return nil } + guard let port = Int(parts[2]), port > 0 else { return nil } + let host = parts[3].hasSuffix(".") ? String(parts[3].dropLast()) : parts[3] + return (host, port) + } + + private static func parseTxtTokens(_ stdout: String) -> [String] { + let lines = stdout.split(whereSeparator: \.isNewline) + var tokens: [String] = [] + for raw in lines { + let line = raw.trimmingCharacters(in: .whitespacesAndNewlines) + if line.isEmpty { continue } + let matches = line.matches(of: /"([^"]*)"/) + for match in matches { + tokens.append(self.unescapeTxt(String(match.1))) + } + } + return tokens + } + + private static func unescapeTxt(_ value: String) -> String { + value + .replacingOccurrences(of: "\\\\", with: "\\") + .replacingOccurrences(of: "\\\"", with: "\"") + .replacingOccurrences(of: "\\n", with: "\n") + } + + private static func mapTxt(tokens: [String]) -> [String: String] { + var out: [String: String] = [:] + for token in tokens { + guard let idx = token.firstIndex(of: "=") else { continue } + let key = String(token[.. Int? { + guard let value else { return nil } + let trimmed = value.trimmingCharacters(in: .whitespacesAndNewlines) + return Int(trimmed) + } + + private static func isTailnetIPv4(_ value: String) -> Bool { + let parts = value.split(separator: ".") + if parts.count != 4 { return false } + let octets = parts.compactMap { Int($0) } + if octets.count != 4 { return false } + let a = octets[0] + let b = octets[1] + return a == 100 && b >= 64 && b <= 127 + } + + private static func decodeDnsSdEscapes(_ value: String) -> String { + var bytes: [UInt8] = [] + var pending = "" + + func flushPending() { + guard !pending.isEmpty else { return } + bytes.append(contentsOf: pending.utf8) + pending = "" + } + + let chars = Array(value) + var i = 0 + while i < chars.count { + let ch = chars[i] + if ch == "\\", i + 3 < chars.count { + let digits = String(chars[(i + 1)...(i + 3)]) + if digits.allSatisfy(\.isNumber), + let byte = UInt8(digits) + { + flushPending() + bytes.append(byte) + i += 4 + continue + } + } + pending.append(ch) + i += 1 + } + flushPending() + + if bytes.isEmpty { return value } + if let decoded = String(bytes: bytes, encoding: .utf8) { + return decoded + } + return value + } +} + +private struct TailscaleStatus: Decodable { + struct Node: Decodable { + let tailscaleIPs: [String]? + + var resolvedIPs: [String] { + self.tailscaleIPs ?? [] + } + + private enum CodingKeys: String, CodingKey { + case tailscaleIPs = "TailscaleIPs" + } + } + + let selfNode: Node? + let peer: [String: Node]? + + private enum CodingKeys: String, CodingKey { + case selfNode = "Self" + case peer = "Peer" + } +} + +extension Collection { + fileprivate var nonEmpty: Self? { isEmpty ? nil : self } +} diff --git a/apps/macos/Sources/OpenClawIPC/IPC.swift b/apps/macos/Sources/OpenClawIPC/IPC.swift new file mode 100644 index 0000000000000000000000000000000000000000..9560699d47fcdbae5d1f75bc52f251ea612a6218 --- /dev/null +++ b/apps/macos/Sources/OpenClawIPC/IPC.swift @@ -0,0 +1,417 @@ +import CoreGraphics +import Foundation + +// MARK: - Capabilities + +public enum Capability: String, Codable, CaseIterable, Sendable { + /// AppleScript / Automation access to control other apps (TCC Automation). + case appleScript + case notifications + case accessibility + case screenRecording + case microphone + case speechRecognition + case camera + case location +} + +public enum CameraFacing: String, Codable, Sendable { + case front + case back +} + +// MARK: - Requests + +/// Notification interruption level (maps to UNNotificationInterruptionLevel) +public enum NotificationPriority: String, Codable, Sendable { + case passive // silent, no wake + case active // default + case timeSensitive // breaks through Focus modes +} + +/// Notification delivery mechanism. +public enum NotificationDelivery: String, Codable, Sendable { + /// Use macOS notification center (UNUserNotificationCenter). + case system + /// Use an in-app overlay/toast (no Notification Center history). + case overlay + /// Prefer system; fall back to overlay when system isn't available. + case auto +} + +// MARK: - Canvas geometry + +/// Optional placement hints for the Canvas panel. +/// Values are in screen coordinates (same as `NSWindow` frame). +public struct CanvasPlacement: Codable, Sendable { + public var x: Double? + public var y: Double? + public var width: Double? + public var height: Double? + + public init(x: Double? = nil, y: Double? = nil, width: Double? = nil, height: Double? = nil) { + self.x = x + self.y = y + self.width = width + self.height = height + } +} + +// MARK: - Canvas show result + +public enum CanvasShowStatus: String, Codable, Sendable { + /// Panel was shown, but no navigation occurred (no target passed and session already existed). + case shown + /// Target was a direct URL (http(s) or file). + case web + /// Local canvas target resolved to an existing file. + case ok + /// Local canvas target did not resolve to a file (404 page). + case notFound + /// Local scaffold fallback (e.g., no index.html present). + case welcome +} + +public struct CanvasShowResult: Codable, Sendable { + /// Session directory on disk (e.g. `~/Library/Application Support/OpenClaw/canvas//`). + public var directory: String + /// Target as provided by the caller (may be nil/empty). + public var target: String? + /// Target actually navigated to (nil when no navigation occurred; defaults to "/" for a newly created session). + public var effectiveTarget: String? + public var status: CanvasShowStatus + /// URL that was loaded (nil when no navigation occurred). + public var url: String? + + public init( + directory: String, + target: String?, + effectiveTarget: String?, + status: CanvasShowStatus, + url: String?) + { + self.directory = directory + self.target = target + self.effectiveTarget = effectiveTarget + self.status = status + self.url = url + } +} + +// MARK: - Canvas A2UI + +public enum CanvasA2UICommand: String, Codable, Sendable { + case pushJSONL + case reset +} + +public enum Request: Sendable { + case notify( + title: String, + body: String, + sound: String?, + priority: NotificationPriority?, + delivery: NotificationDelivery?) + case ensurePermissions([Capability], interactive: Bool) + case runShell( + command: [String], + cwd: String?, + env: [String: String]?, + timeoutSec: Double?, + needsScreenRecording: Bool) + case status + case agent(message: String, thinking: String?, session: String?, deliver: Bool, to: String?) + case rpcStatus + case canvasPresent(session: String, path: String?, placement: CanvasPlacement?) + case canvasHide(session: String) + case canvasEval(session: String, javaScript: String) + case canvasSnapshot(session: String, outPath: String?) + case canvasA2UI(session: String, command: CanvasA2UICommand, jsonl: String?) + case nodeList + case nodeDescribe(nodeId: String) + case nodeInvoke(nodeId: String, command: String, paramsJSON: String?) + case cameraSnap(facing: CameraFacing?, maxWidth: Int?, quality: Double?, outPath: String?) + case cameraClip(facing: CameraFacing?, durationMs: Int?, includeAudio: Bool, outPath: String?) + case screenRecord(screenIndex: Int?, durationMs: Int?, fps: Double?, includeAudio: Bool, outPath: String?) +} + +// MARK: - Responses + +public struct Response: Codable, Sendable { + public var ok: Bool + public var message: String? + /// Optional payload (PNG bytes, stdout text, etc.). + public var payload: Data? + + public init(ok: Bool, message: String? = nil, payload: Data? = nil) { + self.ok = ok + self.message = message + self.payload = payload + } +} + +// MARK: - Codable conformance for Request + +extension Request: Codable { + private enum CodingKeys: String, CodingKey { + case type + case title, body, sound, priority, delivery + case caps, interactive + case command, cwd, env, timeoutSec, needsScreenRecording + case message, thinking, session, deliver, to + case rpcStatus + case path + case javaScript + case outPath + case screenIndex + case fps + case canvasA2UICommand + case jsonl + case facing + case maxWidth + case quality + case durationMs + case includeAudio + case placement + case nodeId + case nodeCommand + case paramsJSON + } + + private enum Kind: String, Codable { + case notify + case ensurePermissions + case runShell + case status + case agent + case rpcStatus + case canvasPresent + case canvasHide + case canvasEval + case canvasSnapshot + case canvasA2UI + case nodeList + case nodeDescribe + case nodeInvoke + case cameraSnap + case cameraClip + case screenRecord + } + + public func encode(to encoder: Encoder) throws { + var container = encoder.container(keyedBy: CodingKeys.self) + switch self { + case let .notify(title, body, sound, priority, delivery): + try container.encode(Kind.notify, forKey: .type) + try container.encode(title, forKey: .title) + try container.encode(body, forKey: .body) + try container.encodeIfPresent(sound, forKey: .sound) + try container.encodeIfPresent(priority, forKey: .priority) + try container.encodeIfPresent(delivery, forKey: .delivery) + + case let .ensurePermissions(caps, interactive): + try container.encode(Kind.ensurePermissions, forKey: .type) + try container.encode(caps, forKey: .caps) + try container.encode(interactive, forKey: .interactive) + + case let .runShell(command, cwd, env, timeoutSec, needsSR): + try container.encode(Kind.runShell, forKey: .type) + try container.encode(command, forKey: .command) + try container.encodeIfPresent(cwd, forKey: .cwd) + try container.encodeIfPresent(env, forKey: .env) + try container.encodeIfPresent(timeoutSec, forKey: .timeoutSec) + try container.encode(needsSR, forKey: .needsScreenRecording) + + case .status: + try container.encode(Kind.status, forKey: .type) + + case let .agent(message, thinking, session, deliver, to): + try container.encode(Kind.agent, forKey: .type) + try container.encode(message, forKey: .message) + try container.encodeIfPresent(thinking, forKey: .thinking) + try container.encodeIfPresent(session, forKey: .session) + try container.encode(deliver, forKey: .deliver) + try container.encodeIfPresent(to, forKey: .to) + + case .rpcStatus: + try container.encode(Kind.rpcStatus, forKey: .type) + + case let .canvasPresent(session, path, placement): + try container.encode(Kind.canvasPresent, forKey: .type) + try container.encode(session, forKey: .session) + try container.encodeIfPresent(path, forKey: .path) + try container.encodeIfPresent(placement, forKey: .placement) + + case let .canvasHide(session): + try container.encode(Kind.canvasHide, forKey: .type) + try container.encode(session, forKey: .session) + + case let .canvasEval(session, javaScript): + try container.encode(Kind.canvasEval, forKey: .type) + try container.encode(session, forKey: .session) + try container.encode(javaScript, forKey: .javaScript) + + case let .canvasSnapshot(session, outPath): + try container.encode(Kind.canvasSnapshot, forKey: .type) + try container.encode(session, forKey: .session) + try container.encodeIfPresent(outPath, forKey: .outPath) + + case let .canvasA2UI(session, command, jsonl): + try container.encode(Kind.canvasA2UI, forKey: .type) + try container.encode(session, forKey: .session) + try container.encode(command, forKey: .canvasA2UICommand) + try container.encodeIfPresent(jsonl, forKey: .jsonl) + + case .nodeList: + try container.encode(Kind.nodeList, forKey: .type) + + case let .nodeDescribe(nodeId): + try container.encode(Kind.nodeDescribe, forKey: .type) + try container.encode(nodeId, forKey: .nodeId) + + case let .nodeInvoke(nodeId, command, paramsJSON): + try container.encode(Kind.nodeInvoke, forKey: .type) + try container.encode(nodeId, forKey: .nodeId) + try container.encode(command, forKey: .nodeCommand) + try container.encodeIfPresent(paramsJSON, forKey: .paramsJSON) + + case let .cameraSnap(facing, maxWidth, quality, outPath): + try container.encode(Kind.cameraSnap, forKey: .type) + try container.encodeIfPresent(facing, forKey: .facing) + try container.encodeIfPresent(maxWidth, forKey: .maxWidth) + try container.encodeIfPresent(quality, forKey: .quality) + try container.encodeIfPresent(outPath, forKey: .outPath) + + case let .cameraClip(facing, durationMs, includeAudio, outPath): + try container.encode(Kind.cameraClip, forKey: .type) + try container.encodeIfPresent(facing, forKey: .facing) + try container.encodeIfPresent(durationMs, forKey: .durationMs) + try container.encode(includeAudio, forKey: .includeAudio) + try container.encodeIfPresent(outPath, forKey: .outPath) + + case let .screenRecord(screenIndex, durationMs, fps, includeAudio, outPath): + try container.encode(Kind.screenRecord, forKey: .type) + try container.encodeIfPresent(screenIndex, forKey: .screenIndex) + try container.encodeIfPresent(durationMs, forKey: .durationMs) + try container.encodeIfPresent(fps, forKey: .fps) + try container.encode(includeAudio, forKey: .includeAudio) + try container.encodeIfPresent(outPath, forKey: .outPath) + } + } + + public init(from decoder: Decoder) throws { + let container = try decoder.container(keyedBy: CodingKeys.self) + let kind = try container.decode(Kind.self, forKey: .type) + switch kind { + case .notify: + let title = try container.decode(String.self, forKey: .title) + let body = try container.decode(String.self, forKey: .body) + let sound = try container.decodeIfPresent(String.self, forKey: .sound) + let priority = try container.decodeIfPresent(NotificationPriority.self, forKey: .priority) + let delivery = try container.decodeIfPresent(NotificationDelivery.self, forKey: .delivery) + self = .notify(title: title, body: body, sound: sound, priority: priority, delivery: delivery) + + case .ensurePermissions: + let caps = try container.decode([Capability].self, forKey: .caps) + let interactive = try container.decode(Bool.self, forKey: .interactive) + self = .ensurePermissions(caps, interactive: interactive) + + case .runShell: + let command = try container.decode([String].self, forKey: .command) + let cwd = try container.decodeIfPresent(String.self, forKey: .cwd) + let env = try container.decodeIfPresent([String: String].self, forKey: .env) + let timeout = try container.decodeIfPresent(Double.self, forKey: .timeoutSec) + let needsSR = try container.decode(Bool.self, forKey: .needsScreenRecording) + self = .runShell(command: command, cwd: cwd, env: env, timeoutSec: timeout, needsScreenRecording: needsSR) + + case .status: + self = .status + + case .agent: + let message = try container.decode(String.self, forKey: .message) + let thinking = try container.decodeIfPresent(String.self, forKey: .thinking) + let session = try container.decodeIfPresent(String.self, forKey: .session) + let deliver = try container.decode(Bool.self, forKey: .deliver) + let to = try container.decodeIfPresent(String.self, forKey: .to) + self = .agent(message: message, thinking: thinking, session: session, deliver: deliver, to: to) + + case .rpcStatus: + self = .rpcStatus + + case .canvasPresent: + let session = try container.decode(String.self, forKey: .session) + let path = try container.decodeIfPresent(String.self, forKey: .path) + let placement = try container.decodeIfPresent(CanvasPlacement.self, forKey: .placement) + self = .canvasPresent(session: session, path: path, placement: placement) + + case .canvasHide: + let session = try container.decode(String.self, forKey: .session) + self = .canvasHide(session: session) + + case .canvasEval: + let session = try container.decode(String.self, forKey: .session) + let javaScript = try container.decode(String.self, forKey: .javaScript) + self = .canvasEval(session: session, javaScript: javaScript) + + case .canvasSnapshot: + let session = try container.decode(String.self, forKey: .session) + let outPath = try container.decodeIfPresent(String.self, forKey: .outPath) + self = .canvasSnapshot(session: session, outPath: outPath) + + case .canvasA2UI: + let session = try container.decode(String.self, forKey: .session) + let command = try container.decode(CanvasA2UICommand.self, forKey: .canvasA2UICommand) + let jsonl = try container.decodeIfPresent(String.self, forKey: .jsonl) + self = .canvasA2UI(session: session, command: command, jsonl: jsonl) + + case .nodeList: + self = .nodeList + + case .nodeDescribe: + let nodeId = try container.decode(String.self, forKey: .nodeId) + self = .nodeDescribe(nodeId: nodeId) + + case .nodeInvoke: + let nodeId = try container.decode(String.self, forKey: .nodeId) + let command = try container.decode(String.self, forKey: .nodeCommand) + let paramsJSON = try container.decodeIfPresent(String.self, forKey: .paramsJSON) + self = .nodeInvoke(nodeId: nodeId, command: command, paramsJSON: paramsJSON) + + case .cameraSnap: + let facing = try container.decodeIfPresent(CameraFacing.self, forKey: .facing) + let maxWidth = try container.decodeIfPresent(Int.self, forKey: .maxWidth) + let quality = try container.decodeIfPresent(Double.self, forKey: .quality) + let outPath = try container.decodeIfPresent(String.self, forKey: .outPath) + self = .cameraSnap(facing: facing, maxWidth: maxWidth, quality: quality, outPath: outPath) + + case .cameraClip: + let facing = try container.decodeIfPresent(CameraFacing.self, forKey: .facing) + let durationMs = try container.decodeIfPresent(Int.self, forKey: .durationMs) + let includeAudio = (try? container.decode(Bool.self, forKey: .includeAudio)) ?? true + let outPath = try container.decodeIfPresent(String.self, forKey: .outPath) + self = .cameraClip(facing: facing, durationMs: durationMs, includeAudio: includeAudio, outPath: outPath) + + case .screenRecord: + let screenIndex = try container.decodeIfPresent(Int.self, forKey: .screenIndex) + let durationMs = try container.decodeIfPresent(Int.self, forKey: .durationMs) + let fps = try container.decodeIfPresent(Double.self, forKey: .fps) + let includeAudio = (try? container.decode(Bool.self, forKey: .includeAudio)) ?? true + let outPath = try container.decodeIfPresent(String.self, forKey: .outPath) + self = .screenRecord( + screenIndex: screenIndex, + durationMs: durationMs, + fps: fps, + includeAudio: includeAudio, + outPath: outPath) + } + } +} + +// Shared transport settings +public let controlSocketPath: String = { + let home = FileManager().homeDirectoryForCurrentUser + let preferred = home + .appendingPathComponent("Library/Application Support/OpenClaw/control.sock") + .path + return preferred +}() diff --git a/apps/macos/Sources/OpenClawMacCLI/ConnectCommand.swift b/apps/macos/Sources/OpenClawMacCLI/ConnectCommand.swift new file mode 100644 index 0000000000000000000000000000000000000000..1c31ce3b051611c2f49e6e04f284fa4b408444f6 --- /dev/null +++ b/apps/macos/Sources/OpenClawMacCLI/ConnectCommand.swift @@ -0,0 +1,353 @@ +import OpenClawKit +import OpenClawProtocol +import Foundation +#if canImport(Darwin) +import Darwin +#endif + +struct ConnectOptions { + var url: String? + var token: String? + var password: String? + var mode: String? + var timeoutMs: Int = 15000 + var json: Bool = false + var probe: Bool = false + var clientId: String = "openclaw-macos" + var clientMode: String = "ui" + var displayName: String? + var role: String = "operator" + var scopes: [String] = ["operator.admin", "operator.approvals", "operator.pairing"] + var help: Bool = false + + static func parse(_ args: [String]) -> ConnectOptions { + var opts = ConnectOptions() + let flagHandlers: [String: (inout ConnectOptions) -> Void] = [ + "-h": { $0.help = true }, + "--help": { $0.help = true }, + "--json": { $0.json = true }, + "--probe": { $0.probe = true }, + ] + let valueHandlers: [String: (inout ConnectOptions, String) -> Void] = [ + "--url": { $0.url = $1 }, + "--token": { $0.token = $1 }, + "--password": { $0.password = $1 }, + "--mode": { $0.mode = $1 }, + "--timeout": { opts, raw in + if let parsed = Int(raw.trimmingCharacters(in: .whitespacesAndNewlines)) { + opts.timeoutMs = max(250, parsed) + } + }, + "--client-id": { $0.clientId = $1 }, + "--client-mode": { $0.clientMode = $1 }, + "--display-name": { $0.displayName = $1 }, + "--role": { $0.role = $1 }, + "--scopes": { opts, raw in + opts.scopes = raw.split(separator: ",").map { $0.trimmingCharacters(in: .whitespacesAndNewlines) } + .filter { !$0.isEmpty } + }, + ] + var i = 0 + while i < args.count { + let arg = args[i] + if let handler = flagHandlers[arg] { + handler(&opts) + i += 1 + continue + } + if let handler = valueHandlers[arg], let value = self.nextValue(args, index: &i) { + handler(&opts, value) + i += 1 + continue + } + i += 1 + } + return opts + } + + private static func nextValue(_ args: [String], index: inout Int) -> String? { + guard index + 1 < args.count else { return nil } + index += 1 + return args[index].trimmingCharacters(in: .whitespacesAndNewlines) + } +} + +struct ConnectOutput: Encodable { + var status: String + var url: String + var mode: String + var role: String + var clientId: String + var clientMode: String + var scopes: [String] + var snapshot: HelloOk? + var health: ProtoAnyCodable? + var error: String? +} + +actor SnapshotStore { + private var value: HelloOk? + + func set(_ snapshot: HelloOk) { + self.value = snapshot + } + + func get() -> HelloOk? { + self.value + } +} + +func runConnect(_ args: [String]) async { + let opts = ConnectOptions.parse(args) + if opts.help { + print(""" + openclaw-mac connect + + Usage: + openclaw-mac connect [--url ] [--token ] [--password ] + [--mode ] [--timeout ] [--probe] [--json] + [--client-id ] [--client-mode ] [--display-name ] + [--role ] [--scopes ] + + Options: + --url Gateway WebSocket URL (overrides config) + --token Gateway token (if required) + --password Gateway password (if required) + --mode Resolve from config: local|remote (default: config or local) + --timeout Request timeout (default: 15000) + --probe Force a fresh health probe + --json Emit JSON + --client-id Override client id (default: openclaw-macos) + --client-mode Override client mode (default: ui) + --display-name Override display name + --role Override role (default: operator) + --scopes Override scopes list + -h, --help Show help + """) + return + } + + let config = loadGatewayConfig() + do { + let endpoint = try resolveGatewayEndpoint(opts: opts, config: config) + let displayName = opts.displayName ?? Host.current().localizedName ?? "OpenClaw macOS Debug CLI" + let connectOptions = GatewayConnectOptions( + role: opts.role, + scopes: opts.scopes, + caps: [], + commands: [], + permissions: [:], + clientId: opts.clientId, + clientMode: opts.clientMode, + clientDisplayName: displayName) + + let snapshotStore = SnapshotStore() + let channel = GatewayChannelActor( + url: endpoint.url, + token: endpoint.token, + password: endpoint.password, + pushHandler: { push in + if case let .snapshot(ok) = push { + await snapshotStore.set(ok) + } + }, + connectOptions: connectOptions) + + let params: [String: KitAnyCodable]? = opts.probe ? ["probe": KitAnyCodable(true)] : nil + let data = try await channel.request( + method: "health", + params: params, + timeoutMs: Double(opts.timeoutMs)) + let health = try? JSONDecoder().decode(ProtoAnyCodable.self, from: data) + let snapshot = await snapshotStore.get() + await channel.shutdown() + + let output = ConnectOutput( + status: "ok", + url: endpoint.url.absoluteString, + mode: endpoint.mode, + role: opts.role, + clientId: opts.clientId, + clientMode: opts.clientMode, + scopes: opts.scopes, + snapshot: snapshot, + health: health, + error: nil) + printConnectOutput(output, json: opts.json) + } catch { + let endpoint = bestEffortEndpoint(opts: opts, config: config) + let fallbackMode = (opts.mode ?? config.mode ?? "local").lowercased() + let output = ConnectOutput( + status: "error", + url: endpoint?.url.absoluteString ?? "unknown", + mode: endpoint?.mode ?? fallbackMode, + role: opts.role, + clientId: opts.clientId, + clientMode: opts.clientMode, + scopes: opts.scopes, + snapshot: nil, + health: nil, + error: error.localizedDescription) + printConnectOutput(output, json: opts.json) + exit(1) + } +} + +private func printConnectOutput(_ output: ConnectOutput, json: Bool) { + if json { + let encoder = JSONEncoder() + encoder.outputFormatting = [.prettyPrinted, .sortedKeys] + if let data = try? encoder.encode(output), + let text = String(data: data, encoding: .utf8) + { + print(text) + } else { + print("{\"error\":\"failed to encode JSON\"}") + } + return + } + + print("OpenClaw macOS Gateway Connect") + print("Status: \(output.status)") + print("URL: \(output.url)") + print("Mode: \(output.mode)") + print("Client: \(output.clientId) (\(output.clientMode))") + print("Role: \(output.role)") + print("Scopes: \(output.scopes.joined(separator: ", "))") + if let snapshot = output.snapshot { + print("Protocol: \(snapshot._protocol)") + if let version = snapshot.server["version"]?.value as? String { + print("Server: \(version)") + } + } + if let health = output.health, + let ok = (health.value as? [String: ProtoAnyCodable])?["ok"]?.value as? Bool + { + print("Health: \(ok ? "ok" : "error")") + } else if output.health != nil { + print("Health: received") + } + if let error = output.error { + print("Error: \(error)") + } +} + +private func resolveGatewayEndpoint(opts: ConnectOptions, config: GatewayConfig) throws -> GatewayEndpoint { + let resolvedMode = (opts.mode ?? config.mode ?? "local").lowercased() + if let raw = opts.url, !raw.isEmpty { + guard let url = URL(string: raw) else { + throw NSError(domain: "Gateway", code: 1, userInfo: [NSLocalizedDescriptionKey: "invalid url: \(raw)"]) + } + return GatewayEndpoint( + url: url, + token: resolvedToken(opts: opts, mode: resolvedMode, config: config), + password: resolvedPassword(opts: opts, mode: resolvedMode, config: config), + mode: resolvedMode) + } + + if resolvedMode == "remote" { + guard let raw = config.remoteUrl?.trimmingCharacters(in: .whitespacesAndNewlines), + !raw.isEmpty + else { + throw NSError( + domain: "Gateway", + code: 1, + userInfo: [NSLocalizedDescriptionKey: "gateway.remote.url is missing"]) + } + guard let url = URL(string: raw) else { + throw NSError(domain: "Gateway", code: 1, userInfo: [NSLocalizedDescriptionKey: "invalid url: \(raw)"]) + } + return GatewayEndpoint( + url: url, + token: resolvedToken(opts: opts, mode: resolvedMode, config: config), + password: resolvedPassword(opts: opts, mode: resolvedMode, config: config), + mode: resolvedMode) + } + + let port = config.port ?? 18789 + let host = resolveLocalHost(bind: config.bind) + guard let url = URL(string: "ws://\(host):\(port)") else { + throw NSError( + domain: "Gateway", + code: 1, + userInfo: [NSLocalizedDescriptionKey: "invalid url: ws://\(host):\(port)"]) + } + return GatewayEndpoint( + url: url, + token: resolvedToken(opts: opts, mode: resolvedMode, config: config), + password: resolvedPassword(opts: opts, mode: resolvedMode, config: config), + mode: resolvedMode) +} + +private func bestEffortEndpoint(opts: ConnectOptions, config: GatewayConfig) -> GatewayEndpoint? { + try? resolveGatewayEndpoint(opts: opts, config: config) +} + +private func resolvedToken(opts: ConnectOptions, mode: String, config: GatewayConfig) -> String? { + if let token = opts.token, !token.isEmpty { return token } + if mode == "remote" { + return config.remoteToken + } + return config.token +} + +private func resolvedPassword(opts: ConnectOptions, mode: String, config: GatewayConfig) -> String? { + if let password = opts.password, !password.isEmpty { return password } + if mode == "remote" { + return config.remotePassword + } + return config.password +} + +private func resolveLocalHost(bind: String?) -> String { + let normalized = (bind ?? "").trimmingCharacters(in: .whitespacesAndNewlines).lowercased() + let tailnetIP = detectTailnetIPv4() + switch normalized { + case "tailnet": + return tailnetIP ?? "127.0.0.1" + default: + return "127.0.0.1" + } +} + +private func detectTailnetIPv4() -> String? { + var addrList: UnsafeMutablePointer? + guard getifaddrs(&addrList) == 0, let first = addrList else { return nil } + defer { freeifaddrs(addrList) } + + for ptr in sequence(first: first, next: { $0.pointee.ifa_next }) { + let flags = Int32(ptr.pointee.ifa_flags) + let isUp = (flags & IFF_UP) != 0 + let isLoopback = (flags & IFF_LOOPBACK) != 0 + let family = ptr.pointee.ifa_addr.pointee.sa_family + if !isUp || isLoopback || family != UInt8(AF_INET) { continue } + + var addr = ptr.pointee.ifa_addr.pointee + var buffer = [CChar](repeating: 0, count: Int(NI_MAXHOST)) + let result = getnameinfo( + &addr, + socklen_t(ptr.pointee.ifa_addr.pointee.sa_len), + &buffer, + socklen_t(buffer.count), + nil, + 0, + NI_NUMERICHOST) + guard result == 0 else { continue } + let len = buffer.prefix { $0 != 0 } + let bytes = len.map { UInt8(bitPattern: $0) } + guard let ip = String(bytes: bytes, encoding: .utf8) else { continue } + if isTailnetIPv4(ip) { return ip } + } + + return nil +} + +private func isTailnetIPv4(_ address: String) -> Bool { + let parts = address.split(separator: ".") + guard parts.count == 4 else { return false } + let octets = parts.compactMap { Int($0) } + guard octets.count == 4 else { return false } + let a = octets[0] + let b = octets[1] + return a == 100 && b >= 64 && b <= 127 +} diff --git a/apps/macos/Sources/OpenClawMacCLI/DiscoverCommand.swift b/apps/macos/Sources/OpenClawMacCLI/DiscoverCommand.swift new file mode 100644 index 0000000000000000000000000000000000000000..09ef2bbc051b2bb9b0d7c855a6b013eb53405749 --- /dev/null +++ b/apps/macos/Sources/OpenClawMacCLI/DiscoverCommand.swift @@ -0,0 +1,149 @@ +import OpenClawDiscovery +import Foundation + +struct DiscoveryOptions { + var timeoutMs: Int = 2000 + var json: Bool = false + var includeLocal: Bool = false + var help: Bool = false + + static func parse(_ args: [String]) -> DiscoveryOptions { + var opts = DiscoveryOptions() + var i = 0 + while i < args.count { + let arg = args[i] + switch arg { + case "-h", "--help": + opts.help = true + case "--json": + opts.json = true + case "--include-local": + opts.includeLocal = true + case "--timeout": + let next = (i + 1 < args.count) ? args[i + 1] : nil + if let next, let parsed = Int(next.trimmingCharacters(in: .whitespacesAndNewlines)) { + opts.timeoutMs = max(100, parsed) + i += 1 + } + default: + break + } + i += 1 + } + return opts + } +} + +struct DiscoveryOutput: Encodable { + struct Gateway: Encodable { + var displayName: String + var lanHost: String? + var tailnetDns: String? + var sshPort: Int + var gatewayPort: Int? + var cliPath: String? + var stableID: String + var debugID: String + var isLocal: Bool + } + + var status: String + var timeoutMs: Int + var includeLocal: Bool + var count: Int + var gateways: [Gateway] +} + +func runDiscover(_ args: [String]) async { + let opts = DiscoveryOptions.parse(args) + if opts.help { + print(""" + openclaw-mac discover + + Usage: + openclaw-mac discover [--timeout ] [--json] [--include-local] + + Options: + --timeout Discovery window in milliseconds (default: 2000) + --json Emit JSON + --include-local Include gateways considered local + -h, --help Show help + """) + return + } + + let displayName = Host.current().localizedName ?? ProcessInfo.processInfo.hostName + let model = await MainActor.run { + GatewayDiscoveryModel( + localDisplayName: displayName, + filterLocalGateways: !opts.includeLocal) + } + + await MainActor.run { + model.start() + } + + let nanos = UInt64(max(100, opts.timeoutMs)) * 1_000_000 + try? await Task.sleep(nanoseconds: nanos) + + let gateways = await MainActor.run { model.gateways } + let status = await MainActor.run { model.statusText } + + await MainActor.run { + model.stop() + } + + if opts.json { + let payload = DiscoveryOutput( + status: status, + timeoutMs: opts.timeoutMs, + includeLocal: opts.includeLocal, + count: gateways.count, + gateways: gateways.map { + DiscoveryOutput.Gateway( + displayName: $0.displayName, + lanHost: $0.lanHost, + tailnetDns: $0.tailnetDns, + sshPort: $0.sshPort, + gatewayPort: $0.gatewayPort, + cliPath: $0.cliPath, + stableID: $0.stableID, + debugID: $0.debugID, + isLocal: $0.isLocal) + }) + let encoder = JSONEncoder() + encoder.outputFormatting = [.prettyPrinted, .sortedKeys] + if let data = try? encoder.encode(payload), + let json = String(data: data, encoding: .utf8) + { + print(json) + } else { + print("{\"error\":\"failed to encode JSON\"}") + } + return + } + + print("Gateway Discovery (macOS NWBrowser)") + print("Status: \(status)") + print("Found \(gateways.count) gateway(s)\(opts.includeLocal ? "" : " (local filtered)")") + if gateways.isEmpty { return } + + for gateway in gateways { + let hosts = [gateway.tailnetDns, gateway.lanHost] + .compactMap { $0?.trimmingCharacters(in: .whitespacesAndNewlines) } + .filter { !$0.isEmpty } + .joined(separator: ", ") + print("- \(gateway.displayName)") + print(" hosts: \(hosts.isEmpty ? "(none)" : hosts)") + print(" ssh: \(gateway.sshPort)") + if let port = gateway.gatewayPort { + print(" gatewayPort: \(port)") + } + if let cliPath = gateway.cliPath { + print(" cliPath: \(cliPath)") + } + print(" isLocal: \(gateway.isLocal)") + print(" stableID: \(gateway.stableID)") + print(" debugID: \(gateway.debugID)") + } +} diff --git a/apps/macos/Sources/OpenClawMacCLI/EntryPoint.swift b/apps/macos/Sources/OpenClawMacCLI/EntryPoint.swift new file mode 100644 index 0000000000000000000000000000000000000000..6cb4880cf914313211ee059ac448ea30fa988a4b --- /dev/null +++ b/apps/macos/Sources/OpenClawMacCLI/EntryPoint.swift @@ -0,0 +1,56 @@ +import Foundation + +private struct RootCommand { + var name: String + var args: [String] +} + +@main +struct OpenClawMacCLI { + static func main() async { + let args = Array(CommandLine.arguments.dropFirst()) + let command = parseRootCommand(args) + switch command?.name { + case nil: + printUsage() + case "-h", "--help", "help": + printUsage() + case "connect": + await runConnect(command?.args ?? []) + case "discover": + await runDiscover(command?.args ?? []) + case "wizard": + await runWizardCommand(command?.args ?? []) + default: + fputs("openclaw-mac: unknown command\n", stderr) + printUsage() + exit(1) + } + } +} + +private func parseRootCommand(_ args: [String]) -> RootCommand? { + guard let first = args.first else { return nil } + return RootCommand(name: first, args: Array(args.dropFirst())) +} + +private func printUsage() { + print(""" + openclaw-mac + + Usage: + openclaw-mac connect [--url ] [--token ] [--password ] + [--mode ] [--timeout ] [--probe] [--json] + [--client-id ] [--client-mode ] [--display-name ] + [--role ] [--scopes ] + openclaw-mac discover [--timeout ] [--json] [--include-local] + openclaw-mac wizard [--url ] [--token ] [--password ] + [--mode ] [--workspace ] [--json] + + Examples: + openclaw-mac connect + openclaw-mac connect --url ws://127.0.0.1:18789 --json + openclaw-mac discover --timeout 3000 --json + openclaw-mac wizard --mode local + """) +} diff --git a/apps/macos/Sources/OpenClawMacCLI/GatewayConfig.swift b/apps/macos/Sources/OpenClawMacCLI/GatewayConfig.swift new file mode 100644 index 0000000000000000000000000000000000000000..c3c963b25315e5432f3c3bb35b8a7d2bb08433ee --- /dev/null +++ b/apps/macos/Sources/OpenClawMacCLI/GatewayConfig.swift @@ -0,0 +1,62 @@ +import Foundation + +struct GatewayConfig { + var mode: String? + var bind: String? + var port: Int? + var remoteUrl: String? + var token: String? + var password: String? + var remoteToken: String? + var remotePassword: String? +} + +struct GatewayEndpoint { + let url: URL + let token: String? + let password: String? + let mode: String +} + +func loadGatewayConfig() -> GatewayConfig { + let home = FileManager().homeDirectoryForCurrentUser + let candidates = [ + home.appendingPathComponent(".openclaw/openclaw.json"), + ] + let url = candidates.first { FileManager().isReadableFile(atPath: $0.path) } ?? candidates[0] + guard let data = try? Data(contentsOf: url) else { return GatewayConfig() } + guard let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any] else { + return GatewayConfig() + } + + var cfg = GatewayConfig() + if let gateway = json["gateway"] as? [String: Any] { + cfg.mode = gateway["mode"] as? String + cfg.bind = gateway["bind"] as? String + cfg.port = gateway["port"] as? Int ?? parseInt(gateway["port"]) + + if let auth = gateway["auth"] as? [String: Any] { + cfg.token = auth["token"] as? String + cfg.password = auth["password"] as? String + } + if let remote = gateway["remote"] as? [String: Any] { + cfg.remoteUrl = remote["url"] as? String + cfg.remoteToken = remote["token"] as? String + cfg.remotePassword = remote["password"] as? String + } + } + return cfg +} + +func parseInt(_ value: Any?) -> Int? { + switch value { + case let number as Int: + number + case let number as Double: + Int(number) + case let raw as String: + Int(raw.trimmingCharacters(in: .whitespacesAndNewlines)) + default: + nil + } +} diff --git a/apps/macos/Sources/OpenClawMacCLI/TypeAliases.swift b/apps/macos/Sources/OpenClawMacCLI/TypeAliases.swift new file mode 100644 index 0000000000000000000000000000000000000000..28b3a7ebdf296e56b97552b287a9b8ca23862028 --- /dev/null +++ b/apps/macos/Sources/OpenClawMacCLI/TypeAliases.swift @@ -0,0 +1,5 @@ +import OpenClawKit +import OpenClawProtocol + +typealias ProtoAnyCodable = OpenClawProtocol.AnyCodable +typealias KitAnyCodable = OpenClawKit.AnyCodable diff --git a/apps/macos/Sources/OpenClawMacCLI/WizardCommand.swift b/apps/macos/Sources/OpenClawMacCLI/WizardCommand.swift new file mode 100644 index 0000000000000000000000000000000000000000..9932b4a15bb5114c0049ee062b8e6a4548692a8f --- /dev/null +++ b/apps/macos/Sources/OpenClawMacCLI/WizardCommand.swift @@ -0,0 +1,547 @@ +import OpenClawKit +import OpenClawProtocol +import Darwin +import Foundation + +struct WizardCliOptions { + var url: String? + var token: String? + var password: String? + var mode: String = "local" + var workspace: String? + var json: Bool = false + var help: Bool = false + + static func parse(_ args: [String]) -> WizardCliOptions { + var opts = WizardCliOptions() + var i = 0 + while i < args.count { + let arg = args[i] + switch arg { + case "-h", "--help": + opts.help = true + case "--json": + opts.json = true + case "--url": + opts.url = self.nextValue(args, index: &i) + case "--token": + opts.token = self.nextValue(args, index: &i) + case "--password": + opts.password = self.nextValue(args, index: &i) + case "--mode": + if let value = nextValue(args, index: &i) { + opts.mode = value + } + case "--workspace": + opts.workspace = self.nextValue(args, index: &i) + default: + break + } + i += 1 + } + return opts + } + + private static func nextValue(_ args: [String], index: inout Int) -> String? { + guard index + 1 < args.count else { return nil } + index += 1 + return args[index].trimmingCharacters(in: .whitespacesAndNewlines) + } +} + +enum WizardCliError: Error, CustomStringConvertible { + case invalidUrl(String) + case missingRemoteUrl + case gatewayError(String) + case decodeError(String) + case cancelled + + var description: String { + switch self { + case let .invalidUrl(raw): "Invalid URL: \(raw)" + case .missingRemoteUrl: "gateway.remote.url is missing" + case let .gatewayError(msg): msg + case let .decodeError(msg): msg + case .cancelled: "Wizard cancelled" + } + } +} + +func runWizardCommand(_ args: [String]) async { + let opts = WizardCliOptions.parse(args) + if opts.help { + print(""" + openclaw-mac wizard + + Usage: + openclaw-mac wizard [--url ] [--token ] [--password ] + [--mode ] [--workspace ] [--json] + + Options: + --url Gateway WebSocket URL (overrides config) + --token Gateway token (if required) + --password Gateway password (if required) + --mode Wizard mode (local|remote). Default: local + --workspace Wizard workspace override + --json Print raw wizard responses + -h, --help Show help + """) + return + } + + let config = loadGatewayConfig() + do { + guard isatty(STDIN_FILENO) != 0 else { + throw WizardCliError.gatewayError("Wizard requires an interactive TTY.") + } + let endpoint = try resolveWizardGatewayEndpoint(opts: opts, config: config) + let client = GatewayWizardClient( + url: endpoint.url, + token: endpoint.token, + password: endpoint.password, + json: opts.json) + try await client.connect() + defer { Task { await client.close() } } + try await runWizard(client: client, opts: opts) + } catch { + fputs("wizard: \(error)\n", stderr) + exit(1) + } +} + +private func resolveWizardGatewayEndpoint(opts: WizardCliOptions, config: GatewayConfig) throws -> GatewayEndpoint { + if let raw = opts.url, !raw.isEmpty { + guard let url = URL(string: raw) else { throw WizardCliError.invalidUrl(raw) } + return GatewayEndpoint( + url: url, + token: resolvedToken(opts: opts, config: config), + password: resolvedPassword(opts: opts, config: config), + mode: (config.mode ?? "local").lowercased()) + } + + let mode = (config.mode ?? "local").lowercased() + if mode == "remote" { + guard let raw = config.remoteUrl?.trimmingCharacters(in: .whitespacesAndNewlines), !raw.isEmpty else { + throw WizardCliError.missingRemoteUrl + } + guard let url = URL(string: raw) else { throw WizardCliError.invalidUrl(raw) } + return GatewayEndpoint( + url: url, + token: resolvedToken(opts: opts, config: config), + password: resolvedPassword(opts: opts, config: config), + mode: mode) + } + + let port = config.port ?? 18789 + let host = "127.0.0.1" + guard let url = URL(string: "ws://\(host):\(port)") else { + throw WizardCliError.invalidUrl("ws://\(host):\(port)") + } + return GatewayEndpoint( + url: url, + token: resolvedToken(opts: opts, config: config), + password: resolvedPassword(opts: opts, config: config), + mode: mode) +} + +private func resolvedToken(opts: WizardCliOptions, config: GatewayConfig) -> String? { + if let token = opts.token, !token.isEmpty { return token } + if (config.mode ?? "local").lowercased() == "remote" { + return config.remoteToken + } + return config.token +} + +private func resolvedPassword(opts: WizardCliOptions, config: GatewayConfig) -> String? { + if let password = opts.password, !password.isEmpty { return password } + if (config.mode ?? "local").lowercased() == "remote" { + return config.remotePassword + } + return config.password +} + +actor GatewayWizardClient { + private enum ConnectChallengeError: Error { + case timeout + } + + private let url: URL + private let token: String? + private let password: String? + private let json: Bool + private let encoder = JSONEncoder() + private let decoder = JSONDecoder() + private let session = URLSession(configuration: .default) + private let connectChallengeTimeoutSeconds: Double = 0.75 + private var task: URLSessionWebSocketTask? + + init(url: URL, token: String?, password: String?, json: Bool) { + self.url = url + self.token = token + self.password = password + self.json = json + } + + func connect() async throws { + let socket = self.session.webSocketTask(with: self.url) + socket.maximumMessageSize = 16 * 1024 * 1024 + socket.resume() + self.task = socket + try await self.sendConnect() + } + + func close() { + self.task?.cancel(with: .goingAway, reason: nil) + self.task = nil + } + + func request(method: String, params: [String: ProtoAnyCodable]?) async throws -> ResponseFrame { + guard let task = self.task else { + throw WizardCliError.gatewayError("gateway not connected") + } + let id = UUID().uuidString + let frame = RequestFrame( + type: "req", + id: id, + method: method, + params: params.map { ProtoAnyCodable($0) }) + let data = try self.encoder.encode(frame) + try await task.send(.data(data)) + + while true { + let message = try await task.receive() + let frame = try decodeFrame(message) + if case let .res(res) = frame, res.id == id { + if res.ok == false { + let msg = (res.error?["message"]?.value as? String) ?? "gateway error" + throw WizardCliError.gatewayError(msg) + } + return res + } + } + } + + func decodePayload(_ response: ResponseFrame, as _: T.Type) throws -> T { + guard let payload = response.payload else { + throw WizardCliError.decodeError("missing payload") + } + let data = try self.encoder.encode(payload) + return try self.decoder.decode(T.self, from: data) + } + + private func decodeFrame(_ message: URLSessionWebSocketTask.Message) throws -> GatewayFrame { + let data: Data? = switch message { + case let .data(data): data + case let .string(text): text.data(using: .utf8) + @unknown default: nil + } + guard let data else { + throw WizardCliError.decodeError("empty gateway response") + } + return try self.decoder.decode(GatewayFrame.self, from: data) + } + + private func sendConnect() async throws { + guard let task = self.task else { + throw WizardCliError.gatewayError("gateway not connected") + } + let osVersion = ProcessInfo.processInfo.operatingSystemVersion + let platform = "macos \(osVersion.majorVersion).\(osVersion.minorVersion).\(osVersion.patchVersion)" + let clientId = "openclaw-macos" + let clientMode = "ui" + let role = "operator" + let scopes: [String] = [] + let client: [String: ProtoAnyCodable] = [ + "id": ProtoAnyCodable(clientId), + "displayName": ProtoAnyCodable(Host.current().localizedName ?? "OpenClaw macOS Wizard CLI"), + "version": ProtoAnyCodable("dev"), + "platform": ProtoAnyCodable(platform), + "deviceFamily": ProtoAnyCodable("Mac"), + "mode": ProtoAnyCodable(clientMode), + "instanceId": ProtoAnyCodable(UUID().uuidString), + ] + + var params: [String: ProtoAnyCodable] = [ + "minProtocol": ProtoAnyCodable(GATEWAY_PROTOCOL_VERSION), + "maxProtocol": ProtoAnyCodable(GATEWAY_PROTOCOL_VERSION), + "client": ProtoAnyCodable(client), + "caps": ProtoAnyCodable([String]()), + "locale": ProtoAnyCodable(Locale.preferredLanguages.first ?? Locale.current.identifier), + "userAgent": ProtoAnyCodable(ProcessInfo.processInfo.operatingSystemVersionString), + "role": ProtoAnyCodable(role), + "scopes": ProtoAnyCodable(scopes), + ] + if let token = self.token { + params["auth"] = ProtoAnyCodable(["token": ProtoAnyCodable(token)]) + } else if let password = self.password { + params["auth"] = ProtoAnyCodable(["password": ProtoAnyCodable(password)]) + } + let connectNonce = try await self.waitForConnectChallenge() + let identity = DeviceIdentityStore.loadOrCreate() + let signedAtMs = Int(Date().timeIntervalSince1970 * 1000) + let scopesValue = scopes.joined(separator: ",") + var payloadParts = [ + connectNonce == nil ? "v1" : "v2", + identity.deviceId, + clientId, + clientMode, + role, + scopesValue, + String(signedAtMs), + self.token ?? "", + ] + if let connectNonce { + payloadParts.append(connectNonce) + } + let payload = payloadParts.joined(separator: "|") + if let signature = DeviceIdentityStore.signPayload(payload, identity: identity), + let publicKey = DeviceIdentityStore.publicKeyBase64Url(identity) + { + var device: [String: ProtoAnyCodable] = [ + "id": ProtoAnyCodable(identity.deviceId), + "publicKey": ProtoAnyCodable(publicKey), + "signature": ProtoAnyCodable(signature), + "signedAt": ProtoAnyCodable(signedAtMs), + ] + if let connectNonce { + device["nonce"] = ProtoAnyCodable(connectNonce) + } + params["device"] = ProtoAnyCodable(device) + } + + let reqId = UUID().uuidString + let frame = RequestFrame( + type: "req", + id: reqId, + method: "connect", + params: ProtoAnyCodable(params)) + let data = try self.encoder.encode(frame) + try await task.send(.data(data)) + + while true { + let message = try await task.receive() + let frameResponse = try decodeFrame(message) + if case let .res(res) = frameResponse, res.id == reqId { + if res.ok == false { + let msg = (res.error?["message"]?.value as? String) ?? "gateway connect failed" + throw WizardCliError.gatewayError(msg) + } + _ = try self.decodePayload(res, as: HelloOk.self) + return + } + } + } + + private func waitForConnectChallenge() async throws -> String? { + guard let task = self.task else { return nil } + do { + return try await AsyncTimeout.withTimeout( + seconds: self.connectChallengeTimeoutSeconds, + onTimeout: { ConnectChallengeError.timeout }, + operation: { + while true { + let message = try await task.receive() + let frame = try await self.decodeFrame(message) + if case let .event(evt) = frame, evt.event == "connect.challenge" { + if let payload = evt.payload?.value as? [String: ProtoAnyCodable], + let nonce = payload["nonce"]?.value as? String + { + return nonce + } + } + } + }) + } catch { + if error is ConnectChallengeError { return nil } + throw error + } + } +} + +private func runWizard(client: GatewayWizardClient, opts: WizardCliOptions) async throws { + var params: [String: ProtoAnyCodable] = [:] + let mode = opts.mode.trimmingCharacters(in: .whitespacesAndNewlines).lowercased() + if mode == "local" || mode == "remote" { + params["mode"] = ProtoAnyCodable(mode) + } + if let workspace = opts.workspace?.trimmingCharacters(in: .whitespacesAndNewlines), !workspace.isEmpty { + params["workspace"] = ProtoAnyCodable(workspace) + } + + let startResponse = try await client.request(method: "wizard.start", params: params) + let startResult = try await client.decodePayload(startResponse, as: WizardStartResult.self) + if opts.json { + dumpResult(startResponse) + } + + let sessionId = startResult.sessionid + var nextResult = WizardNextResult( + done: startResult.done, + step: startResult.step, + status: startResult.status, + error: startResult.error) + + do { + while true { + let status = wizardStatusString(nextResult.status) ?? (nextResult.done ? "done" : "running") + if status == "cancelled" { + print("Wizard cancelled.") + return + } + if status == "error" || (nextResult.done && nextResult.error != nil) { + throw WizardCliError.gatewayError(nextResult.error ?? "wizard error") + } + if status == "done" || nextResult.done { + print("Wizard complete.") + return + } + + if let step = decodeWizardStep(nextResult.step) { + let answer = try promptAnswer(for: step) + var answerPayload: [String: ProtoAnyCodable] = [ + "stepId": ProtoAnyCodable(step.id), + ] + if !(answer is NSNull) { + answerPayload["value"] = ProtoAnyCodable(answer) + } + let response = try await client.request( + method: "wizard.next", + params: [ + "sessionId": ProtoAnyCodable(sessionId), + "answer": ProtoAnyCodable(answerPayload), + ]) + nextResult = try await client.decodePayload(response, as: WizardNextResult.self) + if opts.json { + dumpResult(response) + } + } else { + let response = try await client.request( + method: "wizard.next", + params: ["sessionId": ProtoAnyCodable(sessionId)]) + nextResult = try await client.decodePayload(response, as: WizardNextResult.self) + if opts.json { + dumpResult(response) + } + } + } + } catch WizardCliError.cancelled { + _ = try? await client.request( + method: "wizard.cancel", + params: ["sessionId": ProtoAnyCodable(sessionId)]) + throw WizardCliError.cancelled + } +} + +private func dumpResult(_ response: ResponseFrame) { + guard let payload = response.payload else { + print("{\"error\":\"missing payload\"}") + return + } + let encoder = JSONEncoder() + encoder.outputFormatting = [.prettyPrinted, .sortedKeys] + if let data = try? encoder.encode(payload), let text = String(data: data, encoding: .utf8) { + print(text) + } +} + +private func promptAnswer(for step: WizardStep) throws -> Any { + let type = wizardStepType(step) + if let title = step.title, !title.isEmpty { + print("\n\(title)") + } + if let message = step.message, !message.isEmpty { + print(message) + } + + switch type { + case "note": + _ = try readLineWithPrompt("Continue? (enter)") + return NSNull() + case "progress": + _ = try readLineWithPrompt("Continue? (enter)") + return NSNull() + case "action": + _ = try readLineWithPrompt("Run? (enter)") + return true + case "text": + let initial = anyCodableString(step.initialvalue) + let prompt = step.placeholder ?? "Value" + let value = try readLineWithPrompt("\(prompt)\(initial.isEmpty ? "" : " [\(initial)]")") + let trimmed = value.trimmingCharacters(in: .whitespacesAndNewlines) + return trimmed.isEmpty ? initial : trimmed + case "confirm": + let initial = anyCodableBool(step.initialvalue) + let value = try readLineWithPrompt("Confirm? (y/n) [\(initial ? "y" : "n")]") + let trimmed = value.trimmingCharacters(in: .whitespacesAndNewlines).lowercased() + if trimmed.isEmpty { return initial } + return trimmed == "y" || trimmed == "yes" || trimmed == "true" + case "select": + return try promptSelect(step) + case "multiselect": + return try promptMultiSelect(step) + default: + _ = try readLineWithPrompt("Continue? (enter)") + return NSNull() + } +} + +private func promptSelect(_ step: WizardStep) throws -> Any { + let options = parseWizardOptions(step.options) + guard !options.isEmpty else { return NSNull() } + for (idx, option) in options.enumerated() { + let hint = option.hint?.isEmpty == false ? " — \(option.hint!)" : "" + print(" [\(idx + 1)] \(option.label)\(hint)") + } + let initialIndex = options.firstIndex(where: { anyCodableEqual($0.value, step.initialvalue) }) + let defaultLabel = initialIndex.map { " [\($0 + 1)]" } ?? "" + while true { + let input = try readLineWithPrompt("Select one\(defaultLabel)") + let trimmed = input.trimmingCharacters(in: .whitespacesAndNewlines) + if trimmed.isEmpty, let initialIndex { + return options[initialIndex].value?.value ?? options[initialIndex].label + } + if trimmed.lowercased() == "q" { throw WizardCliError.cancelled } + if let number = Int(trimmed), (1...options.count).contains(number) { + let option = options[number - 1] + return option.value?.value ?? option.label + } + print("Invalid selection.") + } +} + +private func promptMultiSelect(_ step: WizardStep) throws -> [Any] { + let options = parseWizardOptions(step.options) + guard !options.isEmpty else { return [] } + for (idx, option) in options.enumerated() { + let hint = option.hint?.isEmpty == false ? " — \(option.hint!)" : "" + print(" [\(idx + 1)] \(option.label)\(hint)") + } + let initialValues = anyCodableArray(step.initialvalue) + let initialIndices = options.enumerated().compactMap { index, option in + initialValues.contains { anyCodableEqual($0, option.value) } ? index + 1 : nil + } + let defaultLabel = initialIndices.isEmpty ? "" : " [\(initialIndices.map(String.init).joined(separator: ","))]" + while true { + let input = try readLineWithPrompt("Select (comma-separated)\(defaultLabel)") + let trimmed = input.trimmingCharacters(in: .whitespacesAndNewlines) + if trimmed.isEmpty { + return initialIndices.map { options[$0 - 1].value?.value ?? options[$0 - 1].label } + } + if trimmed.lowercased() == "q" { throw WizardCliError.cancelled } + let parts = trimmed.split(separator: ",").map { $0.trimmingCharacters(in: .whitespacesAndNewlines) } + let indices = parts.compactMap { Int($0) }.filter { (1...options.count).contains($0) } + if indices.isEmpty { + print("Invalid selection.") + continue + } + return indices.map { options[$0 - 1].value?.value ?? options[$0 - 1].label } + } +} + +private func readLineWithPrompt(_ prompt: String) throws -> String { + print("\(prompt): ", terminator: "") + guard let line = readLine() else { + throw WizardCliError.cancelled + } + return line +} diff --git a/apps/macos/Sources/OpenClawProtocol/GatewayModels.swift b/apps/macos/Sources/OpenClawProtocol/GatewayModels.swift new file mode 100644 index 0000000000000000000000000000000000000000..9d2ca5ed4ca6579ae06e15904325a4ea80a8216a --- /dev/null +++ b/apps/macos/Sources/OpenClawProtocol/GatewayModels.swift @@ -0,0 +1,2454 @@ +// Generated by scripts/protocol-gen-swift.ts — do not edit by hand +import Foundation + +public let GATEWAY_PROTOCOL_VERSION = 3 + +public enum ErrorCode: String, Codable, Sendable { + case notLinked = "NOT_LINKED" + case notPaired = "NOT_PAIRED" + case agentTimeout = "AGENT_TIMEOUT" + case invalidRequest = "INVALID_REQUEST" + case unavailable = "UNAVAILABLE" +} + +public struct ConnectParams: Codable, Sendable { + public let minprotocol: Int + public let maxprotocol: Int + public let client: [String: AnyCodable] + public let caps: [String]? + public let commands: [String]? + public let permissions: [String: AnyCodable]? + public let pathenv: String? + public let role: String? + public let scopes: [String]? + public let device: [String: AnyCodable]? + public let auth: [String: AnyCodable]? + public let locale: String? + public let useragent: String? + + public init( + minprotocol: Int, + maxprotocol: Int, + client: [String: AnyCodable], + caps: [String]?, + commands: [String]?, + permissions: [String: AnyCodable]?, + pathenv: String?, + role: String?, + scopes: [String]?, + device: [String: AnyCodable]?, + auth: [String: AnyCodable]?, + locale: String?, + useragent: String? + ) { + self.minprotocol = minprotocol + self.maxprotocol = maxprotocol + self.client = client + self.caps = caps + self.commands = commands + self.permissions = permissions + self.pathenv = pathenv + self.role = role + self.scopes = scopes + self.device = device + self.auth = auth + self.locale = locale + self.useragent = useragent + } + private enum CodingKeys: String, CodingKey { + case minprotocol = "minProtocol" + case maxprotocol = "maxProtocol" + case client + case caps + case commands + case permissions + case pathenv = "pathEnv" + case role + case scopes + case device + case auth + case locale + case useragent = "userAgent" + } +} + +public struct HelloOk: Codable, Sendable { + public let type: String + public let _protocol: Int + public let server: [String: AnyCodable] + public let features: [String: AnyCodable] + public let snapshot: Snapshot + public let canvashosturl: String? + public let auth: [String: AnyCodable]? + public let policy: [String: AnyCodable] + + public init( + type: String, + _protocol: Int, + server: [String: AnyCodable], + features: [String: AnyCodable], + snapshot: Snapshot, + canvashosturl: String?, + auth: [String: AnyCodable]?, + policy: [String: AnyCodable] + ) { + self.type = type + self._protocol = _protocol + self.server = server + self.features = features + self.snapshot = snapshot + self.canvashosturl = canvashosturl + self.auth = auth + self.policy = policy + } + private enum CodingKeys: String, CodingKey { + case type + case _protocol = "protocol" + case server + case features + case snapshot + case canvashosturl = "canvasHostUrl" + case auth + case policy + } +} + +public struct RequestFrame: Codable, Sendable { + public let type: String + public let id: String + public let method: String + public let params: AnyCodable? + + public init( + type: String, + id: String, + method: String, + params: AnyCodable? + ) { + self.type = type + self.id = id + self.method = method + self.params = params + } + private enum CodingKeys: String, CodingKey { + case type + case id + case method + case params + } +} + +public struct ResponseFrame: Codable, Sendable { + public let type: String + public let id: String + public let ok: Bool + public let payload: AnyCodable? + public let error: [String: AnyCodable]? + + public init( + type: String, + id: String, + ok: Bool, + payload: AnyCodable?, + error: [String: AnyCodable]? + ) { + self.type = type + self.id = id + self.ok = ok + self.payload = payload + self.error = error + } + private enum CodingKeys: String, CodingKey { + case type + case id + case ok + case payload + case error + } +} + +public struct EventFrame: Codable, Sendable { + public let type: String + public let event: String + public let payload: AnyCodable? + public let seq: Int? + public let stateversion: [String: AnyCodable]? + + public init( + type: String, + event: String, + payload: AnyCodable?, + seq: Int?, + stateversion: [String: AnyCodable]? + ) { + self.type = type + self.event = event + self.payload = payload + self.seq = seq + self.stateversion = stateversion + } + private enum CodingKeys: String, CodingKey { + case type + case event + case payload + case seq + case stateversion = "stateVersion" + } +} + +public struct PresenceEntry: Codable, Sendable { + public let host: String? + public let ip: String? + public let version: String? + public let platform: String? + public let devicefamily: String? + public let modelidentifier: String? + public let mode: String? + public let lastinputseconds: Int? + public let reason: String? + public let tags: [String]? + public let text: String? + public let ts: Int + public let deviceid: String? + public let roles: [String]? + public let scopes: [String]? + public let instanceid: String? + + public init( + host: String?, + ip: String?, + version: String?, + platform: String?, + devicefamily: String?, + modelidentifier: String?, + mode: String?, + lastinputseconds: Int?, + reason: String?, + tags: [String]?, + text: String?, + ts: Int, + deviceid: String?, + roles: [String]?, + scopes: [String]?, + instanceid: String? + ) { + self.host = host + self.ip = ip + self.version = version + self.platform = platform + self.devicefamily = devicefamily + self.modelidentifier = modelidentifier + self.mode = mode + self.lastinputseconds = lastinputseconds + self.reason = reason + self.tags = tags + self.text = text + self.ts = ts + self.deviceid = deviceid + self.roles = roles + self.scopes = scopes + self.instanceid = instanceid + } + private enum CodingKeys: String, CodingKey { + case host + case ip + case version + case platform + case devicefamily = "deviceFamily" + case modelidentifier = "modelIdentifier" + case mode + case lastinputseconds = "lastInputSeconds" + case reason + case tags + case text + case ts + case deviceid = "deviceId" + case roles + case scopes + case instanceid = "instanceId" + } +} + +public struct StateVersion: Codable, Sendable { + public let presence: Int + public let health: Int + + public init( + presence: Int, + health: Int + ) { + self.presence = presence + self.health = health + } + private enum CodingKeys: String, CodingKey { + case presence + case health + } +} + +public struct Snapshot: Codable, Sendable { + public let presence: [PresenceEntry] + public let health: AnyCodable + public let stateversion: StateVersion + public let uptimems: Int + public let configpath: String? + public let statedir: String? + public let sessiondefaults: [String: AnyCodable]? + + public init( + presence: [PresenceEntry], + health: AnyCodable, + stateversion: StateVersion, + uptimems: Int, + configpath: String?, + statedir: String?, + sessiondefaults: [String: AnyCodable]? + ) { + self.presence = presence + self.health = health + self.stateversion = stateversion + self.uptimems = uptimems + self.configpath = configpath + self.statedir = statedir + self.sessiondefaults = sessiondefaults + } + private enum CodingKeys: String, CodingKey { + case presence + case health + case stateversion = "stateVersion" + case uptimems = "uptimeMs" + case configpath = "configPath" + case statedir = "stateDir" + case sessiondefaults = "sessionDefaults" + } +} + +public struct ErrorShape: Codable, Sendable { + public let code: String + public let message: String + public let details: AnyCodable? + public let retryable: Bool? + public let retryafterms: Int? + + public init( + code: String, + message: String, + details: AnyCodable?, + retryable: Bool?, + retryafterms: Int? + ) { + self.code = code + self.message = message + self.details = details + self.retryable = retryable + self.retryafterms = retryafterms + } + private enum CodingKeys: String, CodingKey { + case code + case message + case details + case retryable + case retryafterms = "retryAfterMs" + } +} + +public struct AgentEvent: Codable, Sendable { + public let runid: String + public let seq: Int + public let stream: String + public let ts: Int + public let data: [String: AnyCodable] + + public init( + runid: String, + seq: Int, + stream: String, + ts: Int, + data: [String: AnyCodable] + ) { + self.runid = runid + self.seq = seq + self.stream = stream + self.ts = ts + self.data = data + } + private enum CodingKeys: String, CodingKey { + case runid = "runId" + case seq + case stream + case ts + case data + } +} + +public struct SendParams: Codable, Sendable { + public let to: String + public let message: String + public let mediaurl: String? + public let mediaurls: [String]? + public let gifplayback: Bool? + public let channel: String? + public let accountid: String? + public let sessionkey: String? + public let idempotencykey: String + + public init( + to: String, + message: String, + mediaurl: String?, + mediaurls: [String]?, + gifplayback: Bool?, + channel: String?, + accountid: String?, + sessionkey: String?, + idempotencykey: String + ) { + self.to = to + self.message = message + self.mediaurl = mediaurl + self.mediaurls = mediaurls + self.gifplayback = gifplayback + self.channel = channel + self.accountid = accountid + self.sessionkey = sessionkey + self.idempotencykey = idempotencykey + } + private enum CodingKeys: String, CodingKey { + case to + case message + case mediaurl = "mediaUrl" + case mediaurls = "mediaUrls" + case gifplayback = "gifPlayback" + case channel + case accountid = "accountId" + case sessionkey = "sessionKey" + case idempotencykey = "idempotencyKey" + } +} + +public struct PollParams: Codable, Sendable { + public let to: String + public let question: String + public let options: [String] + public let maxselections: Int? + public let durationhours: Int? + public let channel: String? + public let accountid: String? + public let idempotencykey: String + + public init( + to: String, + question: String, + options: [String], + maxselections: Int?, + durationhours: Int?, + channel: String?, + accountid: String?, + idempotencykey: String + ) { + self.to = to + self.question = question + self.options = options + self.maxselections = maxselections + self.durationhours = durationhours + self.channel = channel + self.accountid = accountid + self.idempotencykey = idempotencykey + } + private enum CodingKeys: String, CodingKey { + case to + case question + case options + case maxselections = "maxSelections" + case durationhours = "durationHours" + case channel + case accountid = "accountId" + case idempotencykey = "idempotencyKey" + } +} + +public struct AgentParams: Codable, Sendable { + public let message: String + public let agentid: String? + public let to: String? + public let replyto: String? + public let sessionid: String? + public let sessionkey: String? + public let thinking: String? + public let deliver: Bool? + public let attachments: [AnyCodable]? + public let channel: String? + public let replychannel: String? + public let accountid: String? + public let replyaccountid: String? + public let threadid: String? + public let groupid: String? + public let groupchannel: String? + public let groupspace: String? + public let timeout: Int? + public let lane: String? + public let extrasystemprompt: String? + public let idempotencykey: String + public let label: String? + public let spawnedby: String? + + public init( + message: String, + agentid: String?, + to: String?, + replyto: String?, + sessionid: String?, + sessionkey: String?, + thinking: String?, + deliver: Bool?, + attachments: [AnyCodable]?, + channel: String?, + replychannel: String?, + accountid: String?, + replyaccountid: String?, + threadid: String?, + groupid: String?, + groupchannel: String?, + groupspace: String?, + timeout: Int?, + lane: String?, + extrasystemprompt: String?, + idempotencykey: String, + label: String?, + spawnedby: String? + ) { + self.message = message + self.agentid = agentid + self.to = to + self.replyto = replyto + self.sessionid = sessionid + self.sessionkey = sessionkey + self.thinking = thinking + self.deliver = deliver + self.attachments = attachments + self.channel = channel + self.replychannel = replychannel + self.accountid = accountid + self.replyaccountid = replyaccountid + self.threadid = threadid + self.groupid = groupid + self.groupchannel = groupchannel + self.groupspace = groupspace + self.timeout = timeout + self.lane = lane + self.extrasystemprompt = extrasystemprompt + self.idempotencykey = idempotencykey + self.label = label + self.spawnedby = spawnedby + } + private enum CodingKeys: String, CodingKey { + case message + case agentid = "agentId" + case to + case replyto = "replyTo" + case sessionid = "sessionId" + case sessionkey = "sessionKey" + case thinking + case deliver + case attachments + case channel + case replychannel = "replyChannel" + case accountid = "accountId" + case replyaccountid = "replyAccountId" + case threadid = "threadId" + case groupid = "groupId" + case groupchannel = "groupChannel" + case groupspace = "groupSpace" + case timeout + case lane + case extrasystemprompt = "extraSystemPrompt" + case idempotencykey = "idempotencyKey" + case label + case spawnedby = "spawnedBy" + } +} + +public struct AgentIdentityParams: Codable, Sendable { + public let agentid: String? + public let sessionkey: String? + + public init( + agentid: String?, + sessionkey: String? + ) { + self.agentid = agentid + self.sessionkey = sessionkey + } + private enum CodingKeys: String, CodingKey { + case agentid = "agentId" + case sessionkey = "sessionKey" + } +} + +public struct AgentIdentityResult: Codable, Sendable { + public let agentid: String + public let name: String? + public let avatar: String? + + public init( + agentid: String, + name: String?, + avatar: String? + ) { + self.agentid = agentid + self.name = name + self.avatar = avatar + } + private enum CodingKeys: String, CodingKey { + case agentid = "agentId" + case name + case avatar + } +} + +public struct AgentWaitParams: Codable, Sendable { + public let runid: String + public let timeoutms: Int? + + public init( + runid: String, + timeoutms: Int? + ) { + self.runid = runid + self.timeoutms = timeoutms + } + private enum CodingKeys: String, CodingKey { + case runid = "runId" + case timeoutms = "timeoutMs" + } +} + +public struct WakeParams: Codable, Sendable { + public let mode: AnyCodable + public let text: String + + public init( + mode: AnyCodable, + text: String + ) { + self.mode = mode + self.text = text + } + private enum CodingKeys: String, CodingKey { + case mode + case text + } +} + +public struct NodePairRequestParams: Codable, Sendable { + public let nodeid: String + public let displayname: String? + public let platform: String? + public let version: String? + public let coreversion: String? + public let uiversion: String? + public let devicefamily: String? + public let modelidentifier: String? + public let caps: [String]? + public let commands: [String]? + public let remoteip: String? + public let silent: Bool? + + public init( + nodeid: String, + displayname: String?, + platform: String?, + version: String?, + coreversion: String?, + uiversion: String?, + devicefamily: String?, + modelidentifier: String?, + caps: [String]?, + commands: [String]?, + remoteip: String?, + silent: Bool? + ) { + self.nodeid = nodeid + self.displayname = displayname + self.platform = platform + self.version = version + self.coreversion = coreversion + self.uiversion = uiversion + self.devicefamily = devicefamily + self.modelidentifier = modelidentifier + self.caps = caps + self.commands = commands + self.remoteip = remoteip + self.silent = silent + } + private enum CodingKeys: String, CodingKey { + case nodeid = "nodeId" + case displayname = "displayName" + case platform + case version + case coreversion = "coreVersion" + case uiversion = "uiVersion" + case devicefamily = "deviceFamily" + case modelidentifier = "modelIdentifier" + case caps + case commands + case remoteip = "remoteIp" + case silent + } +} + +public struct NodePairListParams: Codable, Sendable { +} + +public struct NodePairApproveParams: Codable, Sendable { + public let requestid: String + + public init( + requestid: String + ) { + self.requestid = requestid + } + private enum CodingKeys: String, CodingKey { + case requestid = "requestId" + } +} + +public struct NodePairRejectParams: Codable, Sendable { + public let requestid: String + + public init( + requestid: String + ) { + self.requestid = requestid + } + private enum CodingKeys: String, CodingKey { + case requestid = "requestId" + } +} + +public struct NodePairVerifyParams: Codable, Sendable { + public let nodeid: String + public let token: String + + public init( + nodeid: String, + token: String + ) { + self.nodeid = nodeid + self.token = token + } + private enum CodingKeys: String, CodingKey { + case nodeid = "nodeId" + case token + } +} + +public struct NodeRenameParams: Codable, Sendable { + public let nodeid: String + public let displayname: String + + public init( + nodeid: String, + displayname: String + ) { + self.nodeid = nodeid + self.displayname = displayname + } + private enum CodingKeys: String, CodingKey { + case nodeid = "nodeId" + case displayname = "displayName" + } +} + +public struct NodeListParams: Codable, Sendable { +} + +public struct NodeDescribeParams: Codable, Sendable { + public let nodeid: String + + public init( + nodeid: String + ) { + self.nodeid = nodeid + } + private enum CodingKeys: String, CodingKey { + case nodeid = "nodeId" + } +} + +public struct NodeInvokeParams: Codable, Sendable { + public let nodeid: String + public let command: String + public let params: AnyCodable? + public let timeoutms: Int? + public let idempotencykey: String + + public init( + nodeid: String, + command: String, + params: AnyCodable?, + timeoutms: Int?, + idempotencykey: String + ) { + self.nodeid = nodeid + self.command = command + self.params = params + self.timeoutms = timeoutms + self.idempotencykey = idempotencykey + } + private enum CodingKeys: String, CodingKey { + case nodeid = "nodeId" + case command + case params + case timeoutms = "timeoutMs" + case idempotencykey = "idempotencyKey" + } +} + +public struct NodeInvokeResultParams: Codable, Sendable { + public let id: String + public let nodeid: String + public let ok: Bool + public let payload: AnyCodable? + public let payloadjson: String? + public let error: [String: AnyCodable]? + + public init( + id: String, + nodeid: String, + ok: Bool, + payload: AnyCodable?, + payloadjson: String?, + error: [String: AnyCodable]? + ) { + self.id = id + self.nodeid = nodeid + self.ok = ok + self.payload = payload + self.payloadjson = payloadjson + self.error = error + } + private enum CodingKeys: String, CodingKey { + case id + case nodeid = "nodeId" + case ok + case payload + case payloadjson = "payloadJSON" + case error + } +} + +public struct NodeEventParams: Codable, Sendable { + public let event: String + public let payload: AnyCodable? + public let payloadjson: String? + + public init( + event: String, + payload: AnyCodable?, + payloadjson: String? + ) { + self.event = event + self.payload = payload + self.payloadjson = payloadjson + } + private enum CodingKeys: String, CodingKey { + case event + case payload + case payloadjson = "payloadJSON" + } +} + +public struct NodeInvokeRequestEvent: Codable, Sendable { + public let id: String + public let nodeid: String + public let command: String + public let paramsjson: String? + public let timeoutms: Int? + public let idempotencykey: String? + + public init( + id: String, + nodeid: String, + command: String, + paramsjson: String?, + timeoutms: Int?, + idempotencykey: String? + ) { + self.id = id + self.nodeid = nodeid + self.command = command + self.paramsjson = paramsjson + self.timeoutms = timeoutms + self.idempotencykey = idempotencykey + } + private enum CodingKeys: String, CodingKey { + case id + case nodeid = "nodeId" + case command + case paramsjson = "paramsJSON" + case timeoutms = "timeoutMs" + case idempotencykey = "idempotencyKey" + } +} + +public struct SessionsListParams: Codable, Sendable { + public let limit: Int? + public let activeminutes: Int? + public let includeglobal: Bool? + public let includeunknown: Bool? + public let includederivedtitles: Bool? + public let includelastmessage: Bool? + public let label: String? + public let spawnedby: String? + public let agentid: String? + public let search: String? + + public init( + limit: Int?, + activeminutes: Int?, + includeglobal: Bool?, + includeunknown: Bool?, + includederivedtitles: Bool?, + includelastmessage: Bool?, + label: String?, + spawnedby: String?, + agentid: String?, + search: String? + ) { + self.limit = limit + self.activeminutes = activeminutes + self.includeglobal = includeglobal + self.includeunknown = includeunknown + self.includederivedtitles = includederivedtitles + self.includelastmessage = includelastmessage + self.label = label + self.spawnedby = spawnedby + self.agentid = agentid + self.search = search + } + private enum CodingKeys: String, CodingKey { + case limit + case activeminutes = "activeMinutes" + case includeglobal = "includeGlobal" + case includeunknown = "includeUnknown" + case includederivedtitles = "includeDerivedTitles" + case includelastmessage = "includeLastMessage" + case label + case spawnedby = "spawnedBy" + case agentid = "agentId" + case search + } +} + +public struct SessionsPreviewParams: Codable, Sendable { + public let keys: [String] + public let limit: Int? + public let maxchars: Int? + + public init( + keys: [String], + limit: Int?, + maxchars: Int? + ) { + self.keys = keys + self.limit = limit + self.maxchars = maxchars + } + private enum CodingKeys: String, CodingKey { + case keys + case limit + case maxchars = "maxChars" + } +} + +public struct SessionsResolveParams: Codable, Sendable { + public let key: String? + public let sessionid: String? + public let label: String? + public let agentid: String? + public let spawnedby: String? + public let includeglobal: Bool? + public let includeunknown: Bool? + + public init( + key: String?, + sessionid: String?, + label: String?, + agentid: String?, + spawnedby: String?, + includeglobal: Bool?, + includeunknown: Bool? + ) { + self.key = key + self.sessionid = sessionid + self.label = label + self.agentid = agentid + self.spawnedby = spawnedby + self.includeglobal = includeglobal + self.includeunknown = includeunknown + } + private enum CodingKeys: String, CodingKey { + case key + case sessionid = "sessionId" + case label + case agentid = "agentId" + case spawnedby = "spawnedBy" + case includeglobal = "includeGlobal" + case includeunknown = "includeUnknown" + } +} + +public struct SessionsPatchParams: Codable, Sendable { + public let key: String + public let label: AnyCodable? + public let thinkinglevel: AnyCodable? + public let verboselevel: AnyCodable? + public let reasoninglevel: AnyCodable? + public let responseusage: AnyCodable? + public let elevatedlevel: AnyCodable? + public let exechost: AnyCodable? + public let execsecurity: AnyCodable? + public let execask: AnyCodable? + public let execnode: AnyCodable? + public let model: AnyCodable? + public let spawnedby: AnyCodable? + public let sendpolicy: AnyCodable? + public let groupactivation: AnyCodable? + + public init( + key: String, + label: AnyCodable?, + thinkinglevel: AnyCodable?, + verboselevel: AnyCodable?, + reasoninglevel: AnyCodable?, + responseusage: AnyCodable?, + elevatedlevel: AnyCodable?, + exechost: AnyCodable?, + execsecurity: AnyCodable?, + execask: AnyCodable?, + execnode: AnyCodable?, + model: AnyCodable?, + spawnedby: AnyCodable?, + sendpolicy: AnyCodable?, + groupactivation: AnyCodable? + ) { + self.key = key + self.label = label + self.thinkinglevel = thinkinglevel + self.verboselevel = verboselevel + self.reasoninglevel = reasoninglevel + self.responseusage = responseusage + self.elevatedlevel = elevatedlevel + self.exechost = exechost + self.execsecurity = execsecurity + self.execask = execask + self.execnode = execnode + self.model = model + self.spawnedby = spawnedby + self.sendpolicy = sendpolicy + self.groupactivation = groupactivation + } + private enum CodingKeys: String, CodingKey { + case key + case label + case thinkinglevel = "thinkingLevel" + case verboselevel = "verboseLevel" + case reasoninglevel = "reasoningLevel" + case responseusage = "responseUsage" + case elevatedlevel = "elevatedLevel" + case exechost = "execHost" + case execsecurity = "execSecurity" + case execask = "execAsk" + case execnode = "execNode" + case model + case spawnedby = "spawnedBy" + case sendpolicy = "sendPolicy" + case groupactivation = "groupActivation" + } +} + +public struct SessionsResetParams: Codable, Sendable { + public let key: String + + public init( + key: String + ) { + self.key = key + } + private enum CodingKeys: String, CodingKey { + case key + } +} + +public struct SessionsDeleteParams: Codable, Sendable { + public let key: String + public let deletetranscript: Bool? + + public init( + key: String, + deletetranscript: Bool? + ) { + self.key = key + self.deletetranscript = deletetranscript + } + private enum CodingKeys: String, CodingKey { + case key + case deletetranscript = "deleteTranscript" + } +} + +public struct SessionsCompactParams: Codable, Sendable { + public let key: String + public let maxlines: Int? + + public init( + key: String, + maxlines: Int? + ) { + self.key = key + self.maxlines = maxlines + } + private enum CodingKeys: String, CodingKey { + case key + case maxlines = "maxLines" + } +} + +public struct ConfigGetParams: Codable, Sendable { +} + +public struct ConfigSetParams: Codable, Sendable { + public let raw: String + public let basehash: String? + + public init( + raw: String, + basehash: String? + ) { + self.raw = raw + self.basehash = basehash + } + private enum CodingKeys: String, CodingKey { + case raw + case basehash = "baseHash" + } +} + +public struct ConfigApplyParams: Codable, Sendable { + public let raw: String + public let basehash: String? + public let sessionkey: String? + public let note: String? + public let restartdelayms: Int? + + public init( + raw: String, + basehash: String?, + sessionkey: String?, + note: String?, + restartdelayms: Int? + ) { + self.raw = raw + self.basehash = basehash + self.sessionkey = sessionkey + self.note = note + self.restartdelayms = restartdelayms + } + private enum CodingKeys: String, CodingKey { + case raw + case basehash = "baseHash" + case sessionkey = "sessionKey" + case note + case restartdelayms = "restartDelayMs" + } +} + +public struct ConfigPatchParams: Codable, Sendable { + public let raw: String + public let basehash: String? + public let sessionkey: String? + public let note: String? + public let restartdelayms: Int? + + public init( + raw: String, + basehash: String?, + sessionkey: String?, + note: String?, + restartdelayms: Int? + ) { + self.raw = raw + self.basehash = basehash + self.sessionkey = sessionkey + self.note = note + self.restartdelayms = restartdelayms + } + private enum CodingKeys: String, CodingKey { + case raw + case basehash = "baseHash" + case sessionkey = "sessionKey" + case note + case restartdelayms = "restartDelayMs" + } +} + +public struct ConfigSchemaParams: Codable, Sendable { +} + +public struct ConfigSchemaResponse: Codable, Sendable { + public let schema: AnyCodable + public let uihints: [String: AnyCodable] + public let version: String + public let generatedat: String + + public init( + schema: AnyCodable, + uihints: [String: AnyCodable], + version: String, + generatedat: String + ) { + self.schema = schema + self.uihints = uihints + self.version = version + self.generatedat = generatedat + } + private enum CodingKeys: String, CodingKey { + case schema + case uihints = "uiHints" + case version + case generatedat = "generatedAt" + } +} + +public struct WizardStartParams: Codable, Sendable { + public let mode: AnyCodable? + public let workspace: String? + + public init( + mode: AnyCodable?, + workspace: String? + ) { + self.mode = mode + self.workspace = workspace + } + private enum CodingKeys: String, CodingKey { + case mode + case workspace + } +} + +public struct WizardNextParams: Codable, Sendable { + public let sessionid: String + public let answer: [String: AnyCodable]? + + public init( + sessionid: String, + answer: [String: AnyCodable]? + ) { + self.sessionid = sessionid + self.answer = answer + } + private enum CodingKeys: String, CodingKey { + case sessionid = "sessionId" + case answer + } +} + +public struct WizardCancelParams: Codable, Sendable { + public let sessionid: String + + public init( + sessionid: String + ) { + self.sessionid = sessionid + } + private enum CodingKeys: String, CodingKey { + case sessionid = "sessionId" + } +} + +public struct WizardStatusParams: Codable, Sendable { + public let sessionid: String + + public init( + sessionid: String + ) { + self.sessionid = sessionid + } + private enum CodingKeys: String, CodingKey { + case sessionid = "sessionId" + } +} + +public struct WizardStep: Codable, Sendable { + public let id: String + public let type: AnyCodable + public let title: String? + public let message: String? + public let options: [[String: AnyCodable]]? + public let initialvalue: AnyCodable? + public let placeholder: String? + public let sensitive: Bool? + public let executor: AnyCodable? + + public init( + id: String, + type: AnyCodable, + title: String?, + message: String?, + options: [[String: AnyCodable]]?, + initialvalue: AnyCodable?, + placeholder: String?, + sensitive: Bool?, + executor: AnyCodable? + ) { + self.id = id + self.type = type + self.title = title + self.message = message + self.options = options + self.initialvalue = initialvalue + self.placeholder = placeholder + self.sensitive = sensitive + self.executor = executor + } + private enum CodingKeys: String, CodingKey { + case id + case type + case title + case message + case options + case initialvalue = "initialValue" + case placeholder + case sensitive + case executor + } +} + +public struct WizardNextResult: Codable, Sendable { + public let done: Bool + public let step: [String: AnyCodable]? + public let status: AnyCodable? + public let error: String? + + public init( + done: Bool, + step: [String: AnyCodable]?, + status: AnyCodable?, + error: String? + ) { + self.done = done + self.step = step + self.status = status + self.error = error + } + private enum CodingKeys: String, CodingKey { + case done + case step + case status + case error + } +} + +public struct WizardStartResult: Codable, Sendable { + public let sessionid: String + public let done: Bool + public let step: [String: AnyCodable]? + public let status: AnyCodable? + public let error: String? + + public init( + sessionid: String, + done: Bool, + step: [String: AnyCodable]?, + status: AnyCodable?, + error: String? + ) { + self.sessionid = sessionid + self.done = done + self.step = step + self.status = status + self.error = error + } + private enum CodingKeys: String, CodingKey { + case sessionid = "sessionId" + case done + case step + case status + case error + } +} + +public struct WizardStatusResult: Codable, Sendable { + public let status: AnyCodable + public let error: String? + + public init( + status: AnyCodable, + error: String? + ) { + self.status = status + self.error = error + } + private enum CodingKeys: String, CodingKey { + case status + case error + } +} + +public struct TalkModeParams: Codable, Sendable { + public let enabled: Bool + public let phase: String? + + public init( + enabled: Bool, + phase: String? + ) { + self.enabled = enabled + self.phase = phase + } + private enum CodingKeys: String, CodingKey { + case enabled + case phase + } +} + +public struct ChannelsStatusParams: Codable, Sendable { + public let probe: Bool? + public let timeoutms: Int? + + public init( + probe: Bool?, + timeoutms: Int? + ) { + self.probe = probe + self.timeoutms = timeoutms + } + private enum CodingKeys: String, CodingKey { + case probe + case timeoutms = "timeoutMs" + } +} + +public struct ChannelsStatusResult: Codable, Sendable { + public let ts: Int + public let channelorder: [String] + public let channellabels: [String: AnyCodable] + public let channeldetaillabels: [String: AnyCodable]? + public let channelsystemimages: [String: AnyCodable]? + public let channelmeta: [[String: AnyCodable]]? + public let channels: [String: AnyCodable] + public let channelaccounts: [String: AnyCodable] + public let channeldefaultaccountid: [String: AnyCodable] + + public init( + ts: Int, + channelorder: [String], + channellabels: [String: AnyCodable], + channeldetaillabels: [String: AnyCodable]?, + channelsystemimages: [String: AnyCodable]?, + channelmeta: [[String: AnyCodable]]?, + channels: [String: AnyCodable], + channelaccounts: [String: AnyCodable], + channeldefaultaccountid: [String: AnyCodable] + ) { + self.ts = ts + self.channelorder = channelorder + self.channellabels = channellabels + self.channeldetaillabels = channeldetaillabels + self.channelsystemimages = channelsystemimages + self.channelmeta = channelmeta + self.channels = channels + self.channelaccounts = channelaccounts + self.channeldefaultaccountid = channeldefaultaccountid + } + private enum CodingKeys: String, CodingKey { + case ts + case channelorder = "channelOrder" + case channellabels = "channelLabels" + case channeldetaillabels = "channelDetailLabels" + case channelsystemimages = "channelSystemImages" + case channelmeta = "channelMeta" + case channels + case channelaccounts = "channelAccounts" + case channeldefaultaccountid = "channelDefaultAccountId" + } +} + +public struct ChannelsLogoutParams: Codable, Sendable { + public let channel: String + public let accountid: String? + + public init( + channel: String, + accountid: String? + ) { + self.channel = channel + self.accountid = accountid + } + private enum CodingKeys: String, CodingKey { + case channel + case accountid = "accountId" + } +} + +public struct WebLoginStartParams: Codable, Sendable { + public let force: Bool? + public let timeoutms: Int? + public let verbose: Bool? + public let accountid: String? + + public init( + force: Bool?, + timeoutms: Int?, + verbose: Bool?, + accountid: String? + ) { + self.force = force + self.timeoutms = timeoutms + self.verbose = verbose + self.accountid = accountid + } + private enum CodingKeys: String, CodingKey { + case force + case timeoutms = "timeoutMs" + case verbose + case accountid = "accountId" + } +} + +public struct WebLoginWaitParams: Codable, Sendable { + public let timeoutms: Int? + public let accountid: String? + + public init( + timeoutms: Int?, + accountid: String? + ) { + self.timeoutms = timeoutms + self.accountid = accountid + } + private enum CodingKeys: String, CodingKey { + case timeoutms = "timeoutMs" + case accountid = "accountId" + } +} + +public struct AgentSummary: Codable, Sendable { + public let id: String + public let name: String? + public let identity: [String: AnyCodable]? + + public init( + id: String, + name: String?, + identity: [String: AnyCodable]? + ) { + self.id = id + self.name = name + self.identity = identity + } + private enum CodingKeys: String, CodingKey { + case id + case name + case identity + } +} + +public struct AgentsListParams: Codable, Sendable { +} + +public struct AgentsListResult: Codable, Sendable { + public let defaultid: String + public let mainkey: String + public let scope: AnyCodable + public let agents: [AgentSummary] + + public init( + defaultid: String, + mainkey: String, + scope: AnyCodable, + agents: [AgentSummary] + ) { + self.defaultid = defaultid + self.mainkey = mainkey + self.scope = scope + self.agents = agents + } + private enum CodingKeys: String, CodingKey { + case defaultid = "defaultId" + case mainkey = "mainKey" + case scope + case agents + } +} + +public struct ModelChoice: Codable, Sendable { + public let id: String + public let name: String + public let provider: String + public let contextwindow: Int? + public let reasoning: Bool? + + public init( + id: String, + name: String, + provider: String, + contextwindow: Int?, + reasoning: Bool? + ) { + self.id = id + self.name = name + self.provider = provider + self.contextwindow = contextwindow + self.reasoning = reasoning + } + private enum CodingKeys: String, CodingKey { + case id + case name + case provider + case contextwindow = "contextWindow" + case reasoning + } +} + +public struct ModelsListParams: Codable, Sendable { +} + +public struct ModelsListResult: Codable, Sendable { + public let models: [ModelChoice] + + public init( + models: [ModelChoice] + ) { + self.models = models + } + private enum CodingKeys: String, CodingKey { + case models + } +} + +public struct SkillsStatusParams: Codable, Sendable { +} + +public struct SkillsBinsParams: Codable, Sendable { +} + +public struct SkillsBinsResult: Codable, Sendable { + public let bins: [String] + + public init( + bins: [String] + ) { + self.bins = bins + } + private enum CodingKeys: String, CodingKey { + case bins + } +} + +public struct SkillsInstallParams: Codable, Sendable { + public let name: String + public let installid: String + public let timeoutms: Int? + + public init( + name: String, + installid: String, + timeoutms: Int? + ) { + self.name = name + self.installid = installid + self.timeoutms = timeoutms + } + private enum CodingKeys: String, CodingKey { + case name + case installid = "installId" + case timeoutms = "timeoutMs" + } +} + +public struct SkillsUpdateParams: Codable, Sendable { + public let skillkey: String + public let enabled: Bool? + public let apikey: String? + public let env: [String: AnyCodable]? + + public init( + skillkey: String, + enabled: Bool?, + apikey: String?, + env: [String: AnyCodable]? + ) { + self.skillkey = skillkey + self.enabled = enabled + self.apikey = apikey + self.env = env + } + private enum CodingKeys: String, CodingKey { + case skillkey = "skillKey" + case enabled + case apikey = "apiKey" + case env + } +} + +public struct CronJob: Codable, Sendable { + public let id: String + public let agentid: String? + public let name: String + public let description: String? + public let enabled: Bool + public let deleteafterrun: Bool? + public let createdatms: Int + public let updatedatms: Int + public let schedule: AnyCodable + public let sessiontarget: AnyCodable + public let wakemode: AnyCodable + public let payload: AnyCodable + public let isolation: [String: AnyCodable]? + public let state: [String: AnyCodable] + + public init( + id: String, + agentid: String?, + name: String, + description: String?, + enabled: Bool, + deleteafterrun: Bool?, + createdatms: Int, + updatedatms: Int, + schedule: AnyCodable, + sessiontarget: AnyCodable, + wakemode: AnyCodable, + payload: AnyCodable, + isolation: [String: AnyCodable]?, + state: [String: AnyCodable] + ) { + self.id = id + self.agentid = agentid + self.name = name + self.description = description + self.enabled = enabled + self.deleteafterrun = deleteafterrun + self.createdatms = createdatms + self.updatedatms = updatedatms + self.schedule = schedule + self.sessiontarget = sessiontarget + self.wakemode = wakemode + self.payload = payload + self.isolation = isolation + self.state = state + } + private enum CodingKeys: String, CodingKey { + case id + case agentid = "agentId" + case name + case description + case enabled + case deleteafterrun = "deleteAfterRun" + case createdatms = "createdAtMs" + case updatedatms = "updatedAtMs" + case schedule + case sessiontarget = "sessionTarget" + case wakemode = "wakeMode" + case payload + case isolation + case state + } +} + +public struct CronListParams: Codable, Sendable { + public let includedisabled: Bool? + + public init( + includedisabled: Bool? + ) { + self.includedisabled = includedisabled + } + private enum CodingKeys: String, CodingKey { + case includedisabled = "includeDisabled" + } +} + +public struct CronStatusParams: Codable, Sendable { +} + +public struct CronAddParams: Codable, Sendable { + public let name: String + public let agentid: AnyCodable? + public let description: String? + public let enabled: Bool? + public let deleteafterrun: Bool? + public let schedule: AnyCodable + public let sessiontarget: AnyCodable + public let wakemode: AnyCodable + public let payload: AnyCodable + public let isolation: [String: AnyCodable]? + + public init( + name: String, + agentid: AnyCodable?, + description: String?, + enabled: Bool?, + deleteafterrun: Bool?, + schedule: AnyCodable, + sessiontarget: AnyCodable, + wakemode: AnyCodable, + payload: AnyCodable, + isolation: [String: AnyCodable]? + ) { + self.name = name + self.agentid = agentid + self.description = description + self.enabled = enabled + self.deleteafterrun = deleteafterrun + self.schedule = schedule + self.sessiontarget = sessiontarget + self.wakemode = wakemode + self.payload = payload + self.isolation = isolation + } + private enum CodingKeys: String, CodingKey { + case name + case agentid = "agentId" + case description + case enabled + case deleteafterrun = "deleteAfterRun" + case schedule + case sessiontarget = "sessionTarget" + case wakemode = "wakeMode" + case payload + case isolation + } +} + +public struct CronRunLogEntry: Codable, Sendable { + public let ts: Int + public let jobid: String + public let action: String + public let status: AnyCodable? + public let error: String? + public let summary: String? + public let runatms: Int? + public let durationms: Int? + public let nextrunatms: Int? + + public init( + ts: Int, + jobid: String, + action: String, + status: AnyCodable?, + error: String?, + summary: String?, + runatms: Int?, + durationms: Int?, + nextrunatms: Int? + ) { + self.ts = ts + self.jobid = jobid + self.action = action + self.status = status + self.error = error + self.summary = summary + self.runatms = runatms + self.durationms = durationms + self.nextrunatms = nextrunatms + } + private enum CodingKeys: String, CodingKey { + case ts + case jobid = "jobId" + case action + case status + case error + case summary + case runatms = "runAtMs" + case durationms = "durationMs" + case nextrunatms = "nextRunAtMs" + } +} + +public struct LogsTailParams: Codable, Sendable { + public let cursor: Int? + public let limit: Int? + public let maxbytes: Int? + + public init( + cursor: Int?, + limit: Int?, + maxbytes: Int? + ) { + self.cursor = cursor + self.limit = limit + self.maxbytes = maxbytes + } + private enum CodingKeys: String, CodingKey { + case cursor + case limit + case maxbytes = "maxBytes" + } +} + +public struct LogsTailResult: Codable, Sendable { + public let file: String + public let cursor: Int + public let size: Int + public let lines: [String] + public let truncated: Bool? + public let reset: Bool? + + public init( + file: String, + cursor: Int, + size: Int, + lines: [String], + truncated: Bool?, + reset: Bool? + ) { + self.file = file + self.cursor = cursor + self.size = size + self.lines = lines + self.truncated = truncated + self.reset = reset + } + private enum CodingKeys: String, CodingKey { + case file + case cursor + case size + case lines + case truncated + case reset + } +} + +public struct ExecApprovalsGetParams: Codable, Sendable { +} + +public struct ExecApprovalsSetParams: Codable, Sendable { + public let file: [String: AnyCodable] + public let basehash: String? + + public init( + file: [String: AnyCodable], + basehash: String? + ) { + self.file = file + self.basehash = basehash + } + private enum CodingKeys: String, CodingKey { + case file + case basehash = "baseHash" + } +} + +public struct ExecApprovalsNodeGetParams: Codable, Sendable { + public let nodeid: String + + public init( + nodeid: String + ) { + self.nodeid = nodeid + } + private enum CodingKeys: String, CodingKey { + case nodeid = "nodeId" + } +} + +public struct ExecApprovalsNodeSetParams: Codable, Sendable { + public let nodeid: String + public let file: [String: AnyCodable] + public let basehash: String? + + public init( + nodeid: String, + file: [String: AnyCodable], + basehash: String? + ) { + self.nodeid = nodeid + self.file = file + self.basehash = basehash + } + private enum CodingKeys: String, CodingKey { + case nodeid = "nodeId" + case file + case basehash = "baseHash" + } +} + +public struct ExecApprovalsSnapshot: Codable, Sendable { + public let path: String + public let exists: Bool + public let hash: String + public let file: [String: AnyCodable] + + public init( + path: String, + exists: Bool, + hash: String, + file: [String: AnyCodable] + ) { + self.path = path + self.exists = exists + self.hash = hash + self.file = file + } + private enum CodingKeys: String, CodingKey { + case path + case exists + case hash + case file + } +} + +public struct ExecApprovalRequestParams: Codable, Sendable { + public let id: String? + public let command: String + public let cwd: AnyCodable? + public let host: AnyCodable? + public let security: AnyCodable? + public let ask: AnyCodable? + public let agentid: AnyCodable? + public let resolvedpath: AnyCodable? + public let sessionkey: AnyCodable? + public let timeoutms: Int? + + public init( + id: String?, + command: String, + cwd: AnyCodable?, + host: AnyCodable?, + security: AnyCodable?, + ask: AnyCodable?, + agentid: AnyCodable?, + resolvedpath: AnyCodable?, + sessionkey: AnyCodable?, + timeoutms: Int? + ) { + self.id = id + self.command = command + self.cwd = cwd + self.host = host + self.security = security + self.ask = ask + self.agentid = agentid + self.resolvedpath = resolvedpath + self.sessionkey = sessionkey + self.timeoutms = timeoutms + } + private enum CodingKeys: String, CodingKey { + case id + case command + case cwd + case host + case security + case ask + case agentid = "agentId" + case resolvedpath = "resolvedPath" + case sessionkey = "sessionKey" + case timeoutms = "timeoutMs" + } +} + +public struct ExecApprovalResolveParams: Codable, Sendable { + public let id: String + public let decision: String + + public init( + id: String, + decision: String + ) { + self.id = id + self.decision = decision + } + private enum CodingKeys: String, CodingKey { + case id + case decision + } +} + +public struct DevicePairListParams: Codable, Sendable { +} + +public struct DevicePairApproveParams: Codable, Sendable { + public let requestid: String + + public init( + requestid: String + ) { + self.requestid = requestid + } + private enum CodingKeys: String, CodingKey { + case requestid = "requestId" + } +} + +public struct DevicePairRejectParams: Codable, Sendable { + public let requestid: String + + public init( + requestid: String + ) { + self.requestid = requestid + } + private enum CodingKeys: String, CodingKey { + case requestid = "requestId" + } +} + +public struct DeviceTokenRotateParams: Codable, Sendable { + public let deviceid: String + public let role: String + public let scopes: [String]? + + public init( + deviceid: String, + role: String, + scopes: [String]? + ) { + self.deviceid = deviceid + self.role = role + self.scopes = scopes + } + private enum CodingKeys: String, CodingKey { + case deviceid = "deviceId" + case role + case scopes + } +} + +public struct DeviceTokenRevokeParams: Codable, Sendable { + public let deviceid: String + public let role: String + + public init( + deviceid: String, + role: String + ) { + self.deviceid = deviceid + self.role = role + } + private enum CodingKeys: String, CodingKey { + case deviceid = "deviceId" + case role + } +} + +public struct DevicePairRequestedEvent: Codable, Sendable { + public let requestid: String + public let deviceid: String + public let publickey: String + public let displayname: String? + public let platform: String? + public let clientid: String? + public let clientmode: String? + public let role: String? + public let roles: [String]? + public let scopes: [String]? + public let remoteip: String? + public let silent: Bool? + public let isrepair: Bool? + public let ts: Int + + public init( + requestid: String, + deviceid: String, + publickey: String, + displayname: String?, + platform: String?, + clientid: String?, + clientmode: String?, + role: String?, + roles: [String]?, + scopes: [String]?, + remoteip: String?, + silent: Bool?, + isrepair: Bool?, + ts: Int + ) { + self.requestid = requestid + self.deviceid = deviceid + self.publickey = publickey + self.displayname = displayname + self.platform = platform + self.clientid = clientid + self.clientmode = clientmode + self.role = role + self.roles = roles + self.scopes = scopes + self.remoteip = remoteip + self.silent = silent + self.isrepair = isrepair + self.ts = ts + } + private enum CodingKeys: String, CodingKey { + case requestid = "requestId" + case deviceid = "deviceId" + case publickey = "publicKey" + case displayname = "displayName" + case platform + case clientid = "clientId" + case clientmode = "clientMode" + case role + case roles + case scopes + case remoteip = "remoteIp" + case silent + case isrepair = "isRepair" + case ts + } +} + +public struct DevicePairResolvedEvent: Codable, Sendable { + public let requestid: String + public let deviceid: String + public let decision: String + public let ts: Int + + public init( + requestid: String, + deviceid: String, + decision: String, + ts: Int + ) { + self.requestid = requestid + self.deviceid = deviceid + self.decision = decision + self.ts = ts + } + private enum CodingKeys: String, CodingKey { + case requestid = "requestId" + case deviceid = "deviceId" + case decision + case ts + } +} + +public struct ChatHistoryParams: Codable, Sendable { + public let sessionkey: String + public let limit: Int? + + public init( + sessionkey: String, + limit: Int? + ) { + self.sessionkey = sessionkey + self.limit = limit + } + private enum CodingKeys: String, CodingKey { + case sessionkey = "sessionKey" + case limit + } +} + +public struct ChatSendParams: Codable, Sendable { + public let sessionkey: String + public let message: String + public let thinking: String? + public let deliver: Bool? + public let attachments: [AnyCodable]? + public let timeoutms: Int? + public let idempotencykey: String + + public init( + sessionkey: String, + message: String, + thinking: String?, + deliver: Bool?, + attachments: [AnyCodable]?, + timeoutms: Int?, + idempotencykey: String + ) { + self.sessionkey = sessionkey + self.message = message + self.thinking = thinking + self.deliver = deliver + self.attachments = attachments + self.timeoutms = timeoutms + self.idempotencykey = idempotencykey + } + private enum CodingKeys: String, CodingKey { + case sessionkey = "sessionKey" + case message + case thinking + case deliver + case attachments + case timeoutms = "timeoutMs" + case idempotencykey = "idempotencyKey" + } +} + +public struct ChatAbortParams: Codable, Sendable { + public let sessionkey: String + public let runid: String? + + public init( + sessionkey: String, + runid: String? + ) { + self.sessionkey = sessionkey + self.runid = runid + } + private enum CodingKeys: String, CodingKey { + case sessionkey = "sessionKey" + case runid = "runId" + } +} + +public struct ChatInjectParams: Codable, Sendable { + public let sessionkey: String + public let message: String + public let label: String? + + public init( + sessionkey: String, + message: String, + label: String? + ) { + self.sessionkey = sessionkey + self.message = message + self.label = label + } + private enum CodingKeys: String, CodingKey { + case sessionkey = "sessionKey" + case message + case label + } +} + +public struct ChatEvent: Codable, Sendable { + public let runid: String + public let sessionkey: String + public let seq: Int + public let state: AnyCodable + public let message: AnyCodable? + public let errormessage: String? + public let usage: AnyCodable? + public let stopreason: String? + + public init( + runid: String, + sessionkey: String, + seq: Int, + state: AnyCodable, + message: AnyCodable?, + errormessage: String?, + usage: AnyCodable?, + stopreason: String? + ) { + self.runid = runid + self.sessionkey = sessionkey + self.seq = seq + self.state = state + self.message = message + self.errormessage = errormessage + self.usage = usage + self.stopreason = stopreason + } + private enum CodingKeys: String, CodingKey { + case runid = "runId" + case sessionkey = "sessionKey" + case seq + case state + case message + case errormessage = "errorMessage" + case usage + case stopreason = "stopReason" + } +} + +public struct UpdateRunParams: Codable, Sendable { + public let sessionkey: String? + public let note: String? + public let restartdelayms: Int? + public let timeoutms: Int? + + public init( + sessionkey: String?, + note: String?, + restartdelayms: Int?, + timeoutms: Int? + ) { + self.sessionkey = sessionkey + self.note = note + self.restartdelayms = restartdelayms + self.timeoutms = timeoutms + } + private enum CodingKeys: String, CodingKey { + case sessionkey = "sessionKey" + case note + case restartdelayms = "restartDelayMs" + case timeoutms = "timeoutMs" + } +} + +public struct TickEvent: Codable, Sendable { + public let ts: Int + + public init( + ts: Int + ) { + self.ts = ts + } + private enum CodingKeys: String, CodingKey { + case ts + } +} + +public struct ShutdownEvent: Codable, Sendable { + public let reason: String + public let restartexpectedms: Int? + + public init( + reason: String, + restartexpectedms: Int? + ) { + self.reason = reason + self.restartexpectedms = restartexpectedms + } + private enum CodingKeys: String, CodingKey { + case reason + case restartexpectedms = "restartExpectedMs" + } +} + +public enum GatewayFrame: Codable, Sendable { + case req(RequestFrame) + case res(ResponseFrame) + case event(EventFrame) + case unknown(type: String, raw: [String: AnyCodable]) + + private enum CodingKeys: String, CodingKey { + case type + } + + public init(from decoder: Decoder) throws { + let typeContainer = try decoder.container(keyedBy: CodingKeys.self) + let type = try typeContainer.decode(String.self, forKey: .type) + switch type { + case "req": + self = .req(try RequestFrame(from: decoder)) + case "res": + self = .res(try ResponseFrame(from: decoder)) + case "event": + self = .event(try EventFrame(from: decoder)) + default: + let container = try decoder.singleValueContainer() + let raw = try container.decode([String: AnyCodable].self) + self = .unknown(type: type, raw: raw) + } + } + + public func encode(to encoder: Encoder) throws { + switch self { + case .req(let v): try v.encode(to: encoder) + case .res(let v): try v.encode(to: encoder) + case .event(let v): try v.encode(to: encoder) + case .unknown(_, let raw): + var container = encoder.singleValueContainer() + try container.encode(raw) + } + } + +} diff --git a/apps/macos/Tests/OpenClawIPCTests/AgentEventStoreTests.swift b/apps/macos/Tests/OpenClawIPCTests/AgentEventStoreTests.swift new file mode 100644 index 0000000000000000000000000000000000000000..89754f86a71d5b7bb48951825193029d483467f2 --- /dev/null +++ b/apps/macos/Tests/OpenClawIPCTests/AgentEventStoreTests.swift @@ -0,0 +1,44 @@ +import OpenClawProtocol +import Foundation +import Testing +@testable import OpenClaw + +@Suite +@MainActor +struct AgentEventStoreTests { + @Test + func appendAndClear() { + let store = AgentEventStore() + #expect(store.events.isEmpty) + + store.append(ControlAgentEvent( + runId: "run", + seq: 1, + stream: "test", + ts: 0, + data: [:] as [String: OpenClawProtocol.AnyCodable], + summary: nil)) + #expect(store.events.count == 1) + + store.clear() + #expect(store.events.isEmpty) + } + + @Test + func trimsToMaxEvents() { + let store = AgentEventStore() + for i in 1...401 { + store.append(ControlAgentEvent( + runId: "run", + seq: i, + stream: "test", + ts: Double(i), + data: [:] as [String: OpenClawProtocol.AnyCodable], + summary: nil)) + } + + #expect(store.events.count == 400) + #expect(store.events.first?.seq == 2) + #expect(store.events.last?.seq == 401) + } +} diff --git a/apps/macos/Tests/OpenClawIPCTests/AgentWorkspaceTests.swift b/apps/macos/Tests/OpenClawIPCTests/AgentWorkspaceTests.swift new file mode 100644 index 0000000000000000000000000000000000000000..6d5e4a37efd0ffcdd4b97636dbc438b495374048 --- /dev/null +++ b/apps/macos/Tests/OpenClawIPCTests/AgentWorkspaceTests.swift @@ -0,0 +1,123 @@ +import Foundation +import Testing +@testable import OpenClaw + +@Suite +struct AgentWorkspaceTests { + @Test + func displayPathUsesTildeForHome() { + let home = FileManager().homeDirectoryForCurrentUser + #expect(AgentWorkspace.displayPath(for: home) == "~") + + let inside = home.appendingPathComponent("Projects", isDirectory: true) + #expect(AgentWorkspace.displayPath(for: inside).hasPrefix("~/")) + } + + @Test + func resolveWorkspaceURLExpandsTilde() { + let url = AgentWorkspace.resolveWorkspaceURL(from: "~/tmp") + #expect(url.path.hasSuffix("/tmp")) + } + + @Test + func agentsURLAppendsFilename() { + let root = URL(fileURLWithPath: "/tmp/ws", isDirectory: true) + let url = AgentWorkspace.agentsURL(workspaceURL: root) + #expect(url.lastPathComponent == AgentWorkspace.agentsFilename) + } + + @Test + func bootstrapCreatesAgentsFileWhenMissing() throws { + let tmp = FileManager().temporaryDirectory + .appendingPathComponent("openclaw-ws-\(UUID().uuidString)", isDirectory: true) + defer { try? FileManager().removeItem(at: tmp) } + + let agentsURL = try AgentWorkspace.bootstrap(workspaceURL: tmp) + #expect(FileManager().fileExists(atPath: agentsURL.path)) + + let contents = try String(contentsOf: agentsURL, encoding: .utf8) + #expect(contents.contains("# AGENTS.md")) + + let identityURL = tmp.appendingPathComponent(AgentWorkspace.identityFilename) + let userURL = tmp.appendingPathComponent(AgentWorkspace.userFilename) + let bootstrapURL = tmp.appendingPathComponent(AgentWorkspace.bootstrapFilename) + #expect(FileManager().fileExists(atPath: identityURL.path)) + #expect(FileManager().fileExists(atPath: userURL.path)) + #expect(FileManager().fileExists(atPath: bootstrapURL.path)) + + let second = try AgentWorkspace.bootstrap(workspaceURL: tmp) + #expect(second == agentsURL) + } + + @Test + func bootstrapSafetyRejectsNonEmptyFolderWithoutAgents() throws { + let tmp = FileManager().temporaryDirectory + .appendingPathComponent("openclaw-ws-\(UUID().uuidString)", isDirectory: true) + defer { try? FileManager().removeItem(at: tmp) } + try FileManager().createDirectory(at: tmp, withIntermediateDirectories: true) + let marker = tmp.appendingPathComponent("notes.txt") + try "hello".write(to: marker, atomically: true, encoding: .utf8) + + let result = AgentWorkspace.bootstrapSafety(for: tmp) + switch result { + case .unsafe: + break + case .safe: + #expect(Bool(false), "Expected unsafe bootstrap safety result.") + } + } + + @Test + func bootstrapSafetyAllowsExistingAgentsFile() throws { + let tmp = FileManager().temporaryDirectory + .appendingPathComponent("openclaw-ws-\(UUID().uuidString)", isDirectory: true) + defer { try? FileManager().removeItem(at: tmp) } + try FileManager().createDirectory(at: tmp, withIntermediateDirectories: true) + let agents = tmp.appendingPathComponent(AgentWorkspace.agentsFilename) + try "# AGENTS.md".write(to: agents, atomically: true, encoding: .utf8) + + let result = AgentWorkspace.bootstrapSafety(for: tmp) + switch result { + case .safe: + break + case .unsafe: + #expect(Bool(false), "Expected safe bootstrap safety result.") + } + } + + @Test + func bootstrapSkipsBootstrapFileWhenWorkspaceHasContent() throws { + let tmp = FileManager().temporaryDirectory + .appendingPathComponent("openclaw-ws-\(UUID().uuidString)", isDirectory: true) + defer { try? FileManager().removeItem(at: tmp) } + try FileManager().createDirectory(at: tmp, withIntermediateDirectories: true) + let marker = tmp.appendingPathComponent("notes.txt") + try "hello".write(to: marker, atomically: true, encoding: .utf8) + + _ = try AgentWorkspace.bootstrap(workspaceURL: tmp) + + let bootstrapURL = tmp.appendingPathComponent(AgentWorkspace.bootstrapFilename) + #expect(!FileManager().fileExists(atPath: bootstrapURL.path)) + } + + @Test + func needsBootstrapFalseWhenIdentityAlreadySet() throws { + let tmp = FileManager().temporaryDirectory + .appendingPathComponent("openclaw-ws-\(UUID().uuidString)", isDirectory: true) + defer { try? FileManager().removeItem(at: tmp) } + try FileManager().createDirectory(at: tmp, withIntermediateDirectories: true) + let identityURL = tmp.appendingPathComponent(AgentWorkspace.identityFilename) + try """ + # IDENTITY.md - Agent Identity + + - Name: Clawd + - Creature: Space Lobster + - Vibe: Helpful + - Emoji: lobster + """.write(to: identityURL, atomically: true, encoding: .utf8) + let bootstrapURL = tmp.appendingPathComponent(AgentWorkspace.bootstrapFilename) + try "bootstrap".write(to: bootstrapURL, atomically: true, encoding: .utf8) + + #expect(!AgentWorkspace.needsBootstrap(workspaceURL: tmp)) + } +} diff --git a/apps/macos/Tests/OpenClawIPCTests/AnthropicAuthControlsSmokeTests.swift b/apps/macos/Tests/OpenClawIPCTests/AnthropicAuthControlsSmokeTests.swift new file mode 100644 index 0000000000000000000000000000000000000000..84c618339328bc26c717c535c862aa0bfe5b3f78 --- /dev/null +++ b/apps/macos/Tests/OpenClawIPCTests/AnthropicAuthControlsSmokeTests.swift @@ -0,0 +1,29 @@ +import Testing +@testable import OpenClaw + +@Suite(.serialized) +@MainActor +struct AnthropicAuthControlsSmokeTests { + @Test func anthropicAuthControlsBuildsBodyLocal() { + let pkce = AnthropicOAuth.PKCE(verifier: "verifier", challenge: "challenge") + let view = AnthropicAuthControls( + connectionMode: .local, + oauthStatus: .connected(expiresAtMs: 1_700_000_000_000), + pkce: pkce, + code: "code#state", + statusText: "Detected code", + autoDetectClipboard: false, + autoConnectClipboard: false) + _ = view.body + } + + @Test func anthropicAuthControlsBuildsBodyRemote() { + let view = AnthropicAuthControls( + connectionMode: .remote, + oauthStatus: .missingFile, + pkce: nil, + code: "", + statusText: nil) + _ = view.body + } +} diff --git a/apps/macos/Tests/OpenClawIPCTests/AnthropicAuthResolverTests.swift b/apps/macos/Tests/OpenClawIPCTests/AnthropicAuthResolverTests.swift new file mode 100644 index 0000000000000000000000000000000000000000..c41b7f64be4cf2289efa165067b50a89bc1933c8 --- /dev/null +++ b/apps/macos/Tests/OpenClawIPCTests/AnthropicAuthResolverTests.swift @@ -0,0 +1,52 @@ +import Foundation +import Testing +@testable import OpenClaw + +@Suite +struct AnthropicAuthResolverTests { + @Test + func prefersOAuthFileOverEnv() throws { + let dir = FileManager().temporaryDirectory + .appendingPathComponent("openclaw-oauth-\(UUID().uuidString)", isDirectory: true) + try FileManager().createDirectory(at: dir, withIntermediateDirectories: true) + let oauthFile = dir.appendingPathComponent("oauth.json") + let payload = [ + "anthropic": [ + "type": "oauth", + "refresh": "r1", + "access": "a1", + "expires": 1_234_567_890, + ], + ] + let data = try JSONSerialization.data(withJSONObject: payload, options: [.prettyPrinted, .sortedKeys]) + try data.write(to: oauthFile, options: [.atomic]) + + let status = OpenClawOAuthStore.anthropicOAuthStatus(at: oauthFile) + let mode = AnthropicAuthResolver.resolve(environment: [ + "ANTHROPIC_API_KEY": "sk-ant-ignored", + ], oauthStatus: status) + #expect(mode == .oauthFile) + } + + @Test + func reportsOAuthEnvWhenPresent() { + let mode = AnthropicAuthResolver.resolve(environment: [ + "ANTHROPIC_OAUTH_TOKEN": "token", + ], oauthStatus: .missingFile) + #expect(mode == .oauthEnv) + } + + @Test + func reportsAPIKeyEnvWhenPresent() { + let mode = AnthropicAuthResolver.resolve(environment: [ + "ANTHROPIC_API_KEY": "sk-ant-key", + ], oauthStatus: .missingFile) + #expect(mode == .apiKeyEnv) + } + + @Test + func reportsMissingWhenNothingConfigured() { + let mode = AnthropicAuthResolver.resolve(environment: [:], oauthStatus: .missingFile) + #expect(mode == .missing) + } +} diff --git a/apps/macos/Tests/OpenClawIPCTests/AnthropicOAuthCodeStateTests.swift b/apps/macos/Tests/OpenClawIPCTests/AnthropicOAuthCodeStateTests.swift new file mode 100644 index 0000000000000000000000000000000000000000..3d337c2b279c265f2eda203e24b44ba39af245e9 --- /dev/null +++ b/apps/macos/Tests/OpenClawIPCTests/AnthropicOAuthCodeStateTests.swift @@ -0,0 +1,31 @@ +import Testing +@testable import OpenClaw + +@Suite +struct AnthropicOAuthCodeStateTests { + @Test + func parsesRawToken() { + let parsed = AnthropicOAuthCodeState.parse(from: "abcDEF1234#stateXYZ9876") + #expect(parsed == .init(code: "abcDEF1234", state: "stateXYZ9876")) + } + + @Test + func parsesBacktickedToken() { + let parsed = AnthropicOAuthCodeState.parse(from: "`abcDEF1234#stateXYZ9876`") + #expect(parsed == .init(code: "abcDEF1234", state: "stateXYZ9876")) + } + + @Test + func parsesCallbackURL() { + let raw = "https://console.anthropic.com/oauth/code/callback?code=abcDEF1234&state=stateXYZ9876" + let parsed = AnthropicOAuthCodeState.parse(from: raw) + #expect(parsed == .init(code: "abcDEF1234", state: "stateXYZ9876")) + } + + @Test + func extractsFromSurroundingText() { + let raw = "Paste the code#state value: abcDEF1234#stateXYZ9876 then return." + let parsed = AnthropicOAuthCodeState.parse(from: raw) + #expect(parsed == .init(code: "abcDEF1234", state: "stateXYZ9876")) + } +} diff --git a/apps/macos/Tests/OpenClawIPCTests/AnyCodableEncodingTests.swift b/apps/macos/Tests/OpenClawIPCTests/AnyCodableEncodingTests.swift new file mode 100644 index 0000000000000000000000000000000000000000..98ff08afb1fcc19ab835c4002495a633ff16aad8 --- /dev/null +++ b/apps/macos/Tests/OpenClawIPCTests/AnyCodableEncodingTests.swift @@ -0,0 +1,38 @@ +import OpenClawProtocol +import Foundation +import Testing + +@testable import OpenClaw + +@Suite struct AnyCodableEncodingTests { + @Test func encodesSwiftArrayAndDictionaryValues() throws { + let payload: [String: Any] = [ + "tags": ["node", "ios"], + "meta": ["count": 2], + "null": NSNull(), + ] + + let data = try JSONEncoder().encode(OpenClawProtocol.AnyCodable(payload)) + let obj = try #require(JSONSerialization.jsonObject(with: data) as? [String: Any]) + + #expect(obj["tags"] as? [String] == ["node", "ios"]) + #expect((obj["meta"] as? [String: Any])?["count"] as? Int == 2) + #expect(obj["null"] is NSNull) + } + + @Test func protocolAnyCodableEncodesPrimitiveArrays() throws { + let payload: [String: Any] = [ + "items": [1, "two", NSNull(), ["ok": true]], + ] + + let data = try JSONEncoder().encode(OpenClawProtocol.AnyCodable(payload)) + let obj = try #require(JSONSerialization.jsonObject(with: data) as? [String: Any]) + + let items = try #require(obj["items"] as? [Any]) + #expect(items.count == 4) + #expect(items[0] as? Int == 1) + #expect(items[1] as? String == "two") + #expect(items[2] is NSNull) + #expect((items[3] as? [String: Any])?["ok"] as? Bool == true) + } +} diff --git a/apps/macos/Tests/OpenClawIPCTests/CLIInstallerTests.swift b/apps/macos/Tests/OpenClawIPCTests/CLIInstallerTests.swift new file mode 100644 index 0000000000000000000000000000000000000000..651dfeb4c15c946e100f284645e3ddefac7e3e00 --- /dev/null +++ b/apps/macos/Tests/OpenClawIPCTests/CLIInstallerTests.swift @@ -0,0 +1,34 @@ +import Foundation +import Testing +@testable import OpenClaw + +@Suite(.serialized) +@MainActor +struct CLIInstallerTests { + @Test func installedLocationFindsExecutable() throws { + let fm = FileManager() + let root = fm.temporaryDirectory.appendingPathComponent( + "openclaw-cli-installer-\(UUID().uuidString)") + defer { try? fm.removeItem(at: root) } + + let binDir = root.appendingPathComponent("bin") + try fm.createDirectory(at: binDir, withIntermediateDirectories: true) + let cli = binDir.appendingPathComponent("openclaw") + fm.createFile(atPath: cli.path, contents: Data()) + try fm.setAttributes([.posixPermissions: 0o755], ofItemAtPath: cli.path) + + let found = CLIInstaller.installedLocation( + searchPaths: [binDir.path], + fileManager: fm) + #expect(found == cli.path) + + try fm.removeItem(at: cli) + fm.createFile(atPath: cli.path, contents: Data()) + try fm.setAttributes([.posixPermissions: 0o644], ofItemAtPath: cli.path) + + let missing = CLIInstaller.installedLocation( + searchPaths: [binDir.path], + fileManager: fm) + #expect(missing == nil) + } +} diff --git a/apps/macos/Tests/OpenClawIPCTests/CameraCaptureServiceTests.swift b/apps/macos/Tests/OpenClawIPCTests/CameraCaptureServiceTests.swift new file mode 100644 index 0000000000000000000000000000000000000000..14b5e6058ff41f5de85a5c435bd48f2c9d60eb3d --- /dev/null +++ b/apps/macos/Tests/OpenClawIPCTests/CameraCaptureServiceTests.swift @@ -0,0 +1,21 @@ +import Testing + +@testable import OpenClaw + +@Suite struct CameraCaptureServiceTests { + @Test func normalizeSnapDefaults() { + let res = CameraCaptureService.normalizeSnap(maxWidth: nil, quality: nil) + #expect(res.maxWidth == 1600) + #expect(res.quality == 0.9) + } + + @Test func normalizeSnapClampsValues() { + let low = CameraCaptureService.normalizeSnap(maxWidth: -1, quality: -10) + #expect(low.maxWidth == 1600) + #expect(low.quality == 0.05) + + let high = CameraCaptureService.normalizeSnap(maxWidth: 9999, quality: 10) + #expect(high.maxWidth == 9999) + #expect(high.quality == 1.0) + } +} diff --git a/apps/macos/Tests/OpenClawIPCTests/CameraIPCTests.swift b/apps/macos/Tests/OpenClawIPCTests/CameraIPCTests.swift new file mode 100644 index 0000000000000000000000000000000000000000..a233154af84bd339a7f47d47a38d32b09576a772 --- /dev/null +++ b/apps/macos/Tests/OpenClawIPCTests/CameraIPCTests.swift @@ -0,0 +1,61 @@ +import OpenClawIPC +import Foundation +import Testing + +@Suite struct CameraIPCTests { + @Test func cameraSnapCodableRoundtrip() throws { + let req: Request = .cameraSnap( + facing: .front, + maxWidth: 640, + quality: 0.85, + outPath: "/tmp/test.jpg") + + let data = try JSONEncoder().encode(req) + let decoded = try JSONDecoder().decode(Request.self, from: data) + + switch decoded { + case let .cameraSnap(facing, maxWidth, quality, outPath): + #expect(facing == .front) + #expect(maxWidth == 640) + #expect(quality == 0.85) + #expect(outPath == "/tmp/test.jpg") + default: + Issue.record("expected cameraSnap, got \(decoded)") + } + } + + @Test func cameraClipCodableRoundtrip() throws { + let req: Request = .cameraClip( + facing: .back, + durationMs: 3000, + includeAudio: false, + outPath: "/tmp/test.mp4") + + let data = try JSONEncoder().encode(req) + let decoded = try JSONDecoder().decode(Request.self, from: data) + + switch decoded { + case let .cameraClip(facing, durationMs, includeAudio, outPath): + #expect(facing == .back) + #expect(durationMs == 3000) + #expect(includeAudio == false) + #expect(outPath == "/tmp/test.mp4") + default: + Issue.record("expected cameraClip, got \(decoded)") + } + } + + @Test func cameraClipDefaultsIncludeAudioToTrueWhenMissing() throws { + let json = """ + {"type":"cameraClip","durationMs":1234} + """ + let decoded = try JSONDecoder().decode(Request.self, from: Data(json.utf8)) + switch decoded { + case let .cameraClip(_, durationMs, includeAudio, _): + #expect(durationMs == 1234) + #expect(includeAudio == true) + default: + Issue.record("expected cameraClip, got \(decoded)") + } + } +} diff --git a/apps/macos/Tests/OpenClawIPCTests/CanvasFileWatcherTests.swift b/apps/macos/Tests/OpenClawIPCTests/CanvasFileWatcherTests.swift new file mode 100644 index 0000000000000000000000000000000000000000..3c957161743f959fbf7d6678407724a343f0bdc4 --- /dev/null +++ b/apps/macos/Tests/OpenClawIPCTests/CanvasFileWatcherTests.swift @@ -0,0 +1,78 @@ +import Foundation +import os +import Testing +@testable import OpenClaw + +@Suite(.serialized) struct CanvasFileWatcherTests { + private func makeTempDir() throws -> URL { + let base = URL(fileURLWithPath: NSTemporaryDirectory(), isDirectory: true) + let dir = base.appendingPathComponent("openclaw-canvaswatch-\(UUID().uuidString)", isDirectory: true) + try FileManager().createDirectory(at: dir, withIntermediateDirectories: true) + return dir + } + + @Test func detectsInPlaceFileWrites() async throws { + let dir = try self.makeTempDir() + defer { try? FileManager().removeItem(at: dir) } + + let file = dir.appendingPathComponent("index.html") + try "hello".write(to: file, atomically: false, encoding: .utf8) + + let fired = OSAllocatedUnfairLock(initialState: false) + let waitState = OSAllocatedUnfairLock<(fired: Bool, cont: CheckedContinuation?)>( + initialState: (false, nil)) + + func waitForFire(timeoutNs: UInt64) async -> Bool { + await withTaskGroup(of: Bool.self) { group in + group.addTask { + await withCheckedContinuation { cont in + let resumeImmediately = waitState.withLock { state in + if state.fired { return true } + state.cont = cont + return false + } + if resumeImmediately { + cont.resume() + } + } + return true + } + + group.addTask { + try? await Task.sleep(nanoseconds: timeoutNs) + return false + } + + let result = await group.next() ?? false + group.cancelAll() + return result + } + } + + let watcher = CanvasFileWatcher(url: dir) { + fired.withLock { $0 = true } + let cont = waitState.withLock { state in + state.fired = true + let cont = state.cont + state.cont = nil + return cont + } + cont?.resume() + } + watcher.start() + defer { watcher.stop() } + + // Give the stream a moment to start. + try await Task.sleep(nanoseconds: 150 * 1_000_000) + + // Modify the file in-place (no rename). This used to be missed when only watching the directory vnode. + let handle = try FileHandle(forUpdating: file) + try handle.seekToEnd() + try handle.write(contentsOf: Data(" world".utf8)) + try handle.close() + + let ok = await waitForFire(timeoutNs: 2_000_000_000) + #expect(ok == true) + #expect(fired.withLock { $0 } == true) + } +} diff --git a/apps/macos/Tests/OpenClawIPCTests/CanvasIPCTests.swift b/apps/macos/Tests/OpenClawIPCTests/CanvasIPCTests.swift new file mode 100644 index 0000000000000000000000000000000000000000..b509efd844de8f1d1bdc9706d55a43e99465f22a --- /dev/null +++ b/apps/macos/Tests/OpenClawIPCTests/CanvasIPCTests.swift @@ -0,0 +1,41 @@ +import OpenClawIPC +import Foundation +import Testing + +@Suite struct CanvasIPCTests { + @Test func canvasPresentCodableRoundtrip() throws { + let placement = CanvasPlacement(x: 10, y: 20, width: 640, height: 480) + let req: Request = .canvasPresent(session: "main", path: "/index.html", placement: placement) + + let data = try JSONEncoder().encode(req) + let decoded = try JSONDecoder().decode(Request.self, from: data) + + switch decoded { + case let .canvasPresent(session, path, placement): + #expect(session == "main") + #expect(path == "/index.html") + #expect(placement?.x == 10) + #expect(placement?.y == 20) + #expect(placement?.width == 640) + #expect(placement?.height == 480) + default: + Issue.record("expected canvasPresent, got \(decoded)") + } + } + + @Test func canvasPresentDecodesNilPlacementWhenMissing() throws { + let json = """ + {"type":"canvasPresent","session":"s","path":"/"} + """ + let decoded = try JSONDecoder().decode(Request.self, from: Data(json.utf8)) + + switch decoded { + case let .canvasPresent(session, path, placement): + #expect(session == "s") + #expect(path == "/") + #expect(placement == nil) + default: + Issue.record("expected canvasPresent, got \(decoded)") + } + } +} diff --git a/apps/macos/Tests/OpenClawIPCTests/CanvasWindowSmokeTests.swift b/apps/macos/Tests/OpenClawIPCTests/CanvasWindowSmokeTests.swift new file mode 100644 index 0000000000000000000000000000000000000000..4299ca74fadcbd86f8950a19c054e3fa53667f98 --- /dev/null +++ b/apps/macos/Tests/OpenClawIPCTests/CanvasWindowSmokeTests.swift @@ -0,0 +1,49 @@ +import AppKit +import OpenClawIPC +import Foundation +import Testing +@testable import OpenClaw + +@Suite(.serialized) +@MainActor +struct CanvasWindowSmokeTests { + @Test func panelControllerShowsAndHides() async throws { + let root = FileManager().temporaryDirectory + .appendingPathComponent("openclaw-canvas-test-\(UUID().uuidString)") + try FileManager().createDirectory(at: root, withIntermediateDirectories: true) + defer { try? FileManager().removeItem(at: root) } + + let anchor = { NSRect(x: 200, y: 400, width: 40, height: 40) } + let controller = try CanvasWindowController( + sessionKey: " main/invalid⚡️ ", + root: root, + presentation: .panel(anchorProvider: anchor)) + + #expect(controller.directoryPath.contains("main_invalid__") == true) + + controller.applyPreferredPlacement(CanvasPlacement(x: 120, y: 200, width: 520, height: 680)) + controller.showCanvas(path: "/") + _ = try await controller.eval(javaScript: "1 + 1") + controller.windowDidMove(Notification(name: NSWindow.didMoveNotification)) + controller.windowDidEndLiveResize(Notification(name: NSWindow.didEndLiveResizeNotification)) + controller.hideCanvas() + controller.close() + } + + @Test func windowControllerShowsAndCloses() async throws { + let root = FileManager().temporaryDirectory + .appendingPathComponent("openclaw-canvas-test-\(UUID().uuidString)") + try FileManager().createDirectory(at: root, withIntermediateDirectories: true) + defer { try? FileManager().removeItem(at: root) } + + let controller = try CanvasWindowController( + sessionKey: "main", + root: root, + presentation: .window) + + controller.showCanvas(path: "/") + controller.windowWillClose(Notification(name: NSWindow.willCloseNotification)) + controller.hideCanvas() + controller.close() + } +} diff --git a/apps/macos/Tests/OpenClawIPCTests/ChannelsSettingsSmokeTests.swift b/apps/macos/Tests/OpenClawIPCTests/ChannelsSettingsSmokeTests.swift new file mode 100644 index 0000000000000000000000000000000000000000..8810d12385b9df99d82c472559509b0426d655c4 --- /dev/null +++ b/apps/macos/Tests/OpenClawIPCTests/ChannelsSettingsSmokeTests.swift @@ -0,0 +1,164 @@ +import OpenClawProtocol +import SwiftUI +import Testing +@testable import OpenClaw + +private typealias SnapshotAnyCodable = OpenClaw.AnyCodable + +@Suite(.serialized) +@MainActor +struct ChannelsSettingsSmokeTests { + @Test func channelsSettingsBuildsBodyWithSnapshot() { + let store = ChannelsStore(isPreview: true) + store.snapshot = ChannelsStatusSnapshot( + ts: 1_700_000_000_000, + channelOrder: ["whatsapp", "telegram", "signal", "imessage"], + channelLabels: [ + "whatsapp": "WhatsApp", + "telegram": "Telegram", + "signal": "Signal", + "imessage": "iMessage", + ], + channelDetailLabels: nil, + channelSystemImages: nil, + channelMeta: nil, + channels: [ + "whatsapp": SnapshotAnyCodable([ + "configured": true, + "linked": true, + "authAgeMs": 86_400_000, + "self": ["e164": "+15551234567"], + "running": true, + "connected": false, + "lastConnectedAt": 1_700_000_000_000, + "lastDisconnect": [ + "at": 1_700_000_050_000, + "status": 401, + "error": "logged out", + "loggedOut": true, + ], + "reconnectAttempts": 2, + "lastMessageAt": 1_700_000_060_000, + "lastEventAt": 1_700_000_060_000, + "lastError": "needs login", + ]), + "telegram": SnapshotAnyCodable([ + "configured": true, + "tokenSource": "env", + "running": true, + "mode": "polling", + "lastStartAt": 1_700_000_000_000, + "probe": [ + "ok": true, + "status": 200, + "elapsedMs": 120, + "bot": ["id": 123, "username": "openclawbot"], + "webhook": ["url": "https://example.com/hook", "hasCustomCert": false], + ], + "lastProbeAt": 1_700_000_050_000, + ]), + "signal": SnapshotAnyCodable([ + "configured": true, + "baseUrl": "http://127.0.0.1:8080", + "running": true, + "lastStartAt": 1_700_000_000_000, + "probe": [ + "ok": true, + "status": 200, + "elapsedMs": 140, + "version": "0.12.4", + ], + "lastProbeAt": 1_700_000_050_000, + ]), + "imessage": SnapshotAnyCodable([ + "configured": false, + "running": false, + "lastError": "not configured", + "probe": ["ok": false, "error": "imsg not found (imsg)"], + "lastProbeAt": 1_700_000_050_000, + ]), + ], + channelAccounts: [:], + channelDefaultAccountId: [ + "whatsapp": "default", + "telegram": "default", + "signal": "default", + "imessage": "default", + ]) + + store.whatsappLoginMessage = "Scan QR" + store.whatsappLoginQrDataUrl = + "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMB/ay7pS8AAAAASUVORK5CYII=" + + let view = ChannelsSettings(store: store) + _ = view.body + } + + @Test func channelsSettingsBuildsBodyWithoutSnapshot() { + let store = ChannelsStore(isPreview: true) + store.snapshot = ChannelsStatusSnapshot( + ts: 1_700_000_000_000, + channelOrder: ["whatsapp", "telegram", "signal", "imessage"], + channelLabels: [ + "whatsapp": "WhatsApp", + "telegram": "Telegram", + "signal": "Signal", + "imessage": "iMessage", + ], + channelDetailLabels: nil, + channelSystemImages: nil, + channelMeta: nil, + channels: [ + "whatsapp": SnapshotAnyCodable([ + "configured": false, + "linked": false, + "running": false, + "connected": false, + "reconnectAttempts": 0, + ]), + "telegram": SnapshotAnyCodable([ + "configured": false, + "running": false, + "lastError": "bot missing", + "probe": [ + "ok": false, + "status": 403, + "error": "unauthorized", + "elapsedMs": 120, + ], + "lastProbeAt": 1_700_000_100_000, + ]), + "signal": SnapshotAnyCodable([ + "configured": false, + "baseUrl": "http://127.0.0.1:8080", + "running": false, + "lastError": "not configured", + "probe": [ + "ok": false, + "status": 404, + "error": "unreachable", + "elapsedMs": 200, + ], + "lastProbeAt": 1_700_000_200_000, + ]), + "imessage": SnapshotAnyCodable([ + "configured": false, + "running": false, + "lastError": "not configured", + "cliPath": "imsg", + "probe": ["ok": false, "error": "imsg not found (imsg)"], + "lastProbeAt": 1_700_000_200_000, + ]), + ], + channelAccounts: [:], + channelDefaultAccountId: [ + "whatsapp": "default", + "telegram": "default", + "signal": "default", + "imessage": "default", + ]) + + let view = ChannelsSettings(store: store) + _ = view.body + } +} diff --git a/apps/macos/Tests/OpenClawIPCTests/CommandResolverTests.swift b/apps/macos/Tests/OpenClawIPCTests/CommandResolverTests.swift new file mode 100644 index 0000000000000000000000000000000000000000..7a71bc08b6ea312be90942833b185c1ba63174d1 --- /dev/null +++ b/apps/macos/Tests/OpenClawIPCTests/CommandResolverTests.swift @@ -0,0 +1,171 @@ +import Darwin +import Foundation +import Testing +@testable import OpenClaw + +@Suite(.serialized) struct CommandResolverTests { + private func makeDefaults() -> UserDefaults { + // Use a unique suite to avoid cross-suite concurrency on UserDefaults.standard. + UserDefaults(suiteName: "CommandResolverTests.\(UUID().uuidString)")! + } + + private func makeTempDir() throws -> URL { + let base = URL(fileURLWithPath: NSTemporaryDirectory(), isDirectory: true) + let dir = base.appendingPathComponent(UUID().uuidString, isDirectory: true) + try FileManager().createDirectory(at: dir, withIntermediateDirectories: true) + return dir + } + + private func makeExec(at path: URL) throws { + try FileManager().createDirectory( + at: path.deletingLastPathComponent(), + withIntermediateDirectories: true) + FileManager().createFile(atPath: path.path, contents: Data("echo ok\n".utf8)) + try FileManager().setAttributes([.posixPermissions: 0o755], ofItemAtPath: path.path) + } + + @Test func prefersOpenClawBinary() async throws { + let defaults = self.makeDefaults() + defaults.set(AppState.ConnectionMode.local.rawValue, forKey: connectionModeKey) + + let tmp = try makeTempDir() + CommandResolver.setProjectRoot(tmp.path) + + let openclawPath = tmp.appendingPathComponent("node_modules/.bin/openclaw") + try self.makeExec(at: openclawPath) + + let cmd = CommandResolver.openclawCommand(subcommand: "gateway", defaults: defaults, configRoot: [:]) + #expect(cmd.prefix(2).elementsEqual([openclawPath.path, "gateway"])) + } + + @Test func fallsBackToNodeAndScript() async throws { + let defaults = self.makeDefaults() + defaults.set(AppState.ConnectionMode.local.rawValue, forKey: connectionModeKey) + + let tmp = try makeTempDir() + CommandResolver.setProjectRoot(tmp.path) + + let nodePath = tmp.appendingPathComponent("node_modules/.bin/node") + let scriptPath = tmp.appendingPathComponent("bin/openclaw.js") + try self.makeExec(at: nodePath) + try "#!/bin/sh\necho v22.0.0\n".write(to: nodePath, atomically: true, encoding: .utf8) + try FileManager().setAttributes([.posixPermissions: 0o755], ofItemAtPath: nodePath.path) + try self.makeExec(at: scriptPath) + + let cmd = CommandResolver.openclawCommand( + subcommand: "rpc", + defaults: defaults, + configRoot: [:], + searchPaths: [tmp.appendingPathComponent("node_modules/.bin").path]) + + #expect(cmd.count >= 3) + if cmd.count >= 3 { + #expect(cmd[0] == nodePath.path) + #expect(cmd[1] == scriptPath.path) + #expect(cmd[2] == "rpc") + } + } + + @Test func fallsBackToPnpm() async throws { + let defaults = self.makeDefaults() + defaults.set(AppState.ConnectionMode.local.rawValue, forKey: connectionModeKey) + + let tmp = try makeTempDir() + CommandResolver.setProjectRoot(tmp.path) + + let pnpmPath = tmp.appendingPathComponent("node_modules/.bin/pnpm") + try self.makeExec(at: pnpmPath) + + let cmd = CommandResolver.openclawCommand(subcommand: "rpc", defaults: defaults, configRoot: [:]) + + #expect(cmd.prefix(4).elementsEqual([pnpmPath.path, "--silent", "openclaw", "rpc"])) + } + + @Test func pnpmKeepsExtraArgsAfterSubcommand() async throws { + let defaults = self.makeDefaults() + defaults.set(AppState.ConnectionMode.local.rawValue, forKey: connectionModeKey) + + let tmp = try makeTempDir() + CommandResolver.setProjectRoot(tmp.path) + + let pnpmPath = tmp.appendingPathComponent("node_modules/.bin/pnpm") + try self.makeExec(at: pnpmPath) + + let cmd = CommandResolver.openclawCommand( + subcommand: "health", + extraArgs: ["--json", "--timeout", "5"], + defaults: defaults, + configRoot: [:]) + + #expect(cmd.prefix(5).elementsEqual([pnpmPath.path, "--silent", "openclaw", "health", "--json"])) + #expect(cmd.suffix(2).elementsEqual(["--timeout", "5"])) + } + + @Test func preferredPathsStartWithProjectNodeBins() async throws { + let tmp = try makeTempDir() + CommandResolver.setProjectRoot(tmp.path) + + let first = CommandResolver.preferredPaths().first + #expect(first == tmp.appendingPathComponent("node_modules/.bin").path) + } + + @Test func buildsSSHCommandForRemoteMode() async throws { + let defaults = self.makeDefaults() + defaults.set(AppState.ConnectionMode.remote.rawValue, forKey: connectionModeKey) + defaults.set("openclaw@example.com:2222", forKey: remoteTargetKey) + defaults.set("/tmp/id_ed25519", forKey: remoteIdentityKey) + defaults.set("/srv/openclaw", forKey: remoteProjectRootKey) + + let cmd = CommandResolver.openclawCommand( + subcommand: "status", + extraArgs: ["--json"], + defaults: defaults, + configRoot: [:]) + + #expect(cmd.first == "/usr/bin/ssh") + if let marker = cmd.firstIndex(of: "--") { + #expect(cmd[marker + 1] == "openclaw@example.com") + } else { + #expect(Bool(false)) + } + #expect(cmd.contains("-i")) + #expect(cmd.contains("/tmp/id_ed25519")) + if let script = cmd.last { + #expect(script.contains("PRJ='/srv/openclaw'")) + #expect(script.contains("cd \"$PRJ\"")) + #expect(script.contains("openclaw")) + #expect(script.contains("status")) + #expect(script.contains("--json")) + #expect(script.contains("CLI=")) + } + } + + @Test func rejectsUnsafeSSHTargets() async throws { + #expect(CommandResolver.parseSSHTarget("-oProxyCommand=calc") == nil) + #expect(CommandResolver.parseSSHTarget("host:-oProxyCommand=calc") == nil) + #expect(CommandResolver.parseSSHTarget("user@host:2222")?.port == 2222) + } + + @Test func configRootLocalOverridesRemoteDefaults() async throws { + let defaults = self.makeDefaults() + defaults.set(AppState.ConnectionMode.remote.rawValue, forKey: connectionModeKey) + defaults.set("openclaw@example.com:2222", forKey: remoteTargetKey) + + let tmp = try makeTempDir() + CommandResolver.setProjectRoot(tmp.path) + + let openclawPath = tmp.appendingPathComponent("node_modules/.bin/openclaw") + try self.makeExec(at: openclawPath) + + let cmd = CommandResolver.openclawCommand( + subcommand: "daemon", + defaults: defaults, + configRoot: ["gateway": ["mode": "local"]]) + + #expect(cmd.first == openclawPath.path) + #expect(cmd.count >= 2) + if cmd.count >= 2 { + #expect(cmd[1] == "daemon") + } + } +} diff --git a/apps/macos/Tests/OpenClawIPCTests/ConfigStoreTests.swift b/apps/macos/Tests/OpenClawIPCTests/ConfigStoreTests.swift new file mode 100644 index 0000000000000000000000000000000000000000..50f72241dd8ee8cc0d11e523aad8f52996c60379 --- /dev/null +++ b/apps/macos/Tests/OpenClawIPCTests/ConfigStoreTests.swift @@ -0,0 +1,68 @@ +import Testing +@testable import OpenClaw + +@Suite(.serialized) +@MainActor +struct ConfigStoreTests { + @Test func loadUsesRemoteInRemoteMode() async { + var localHit = false + var remoteHit = false + await ConfigStore._testSetOverrides(.init( + isRemoteMode: { true }, + loadLocal: { localHit = true; return ["local": true] }, + loadRemote: { remoteHit = true; return ["remote": true] })) + + let result = await ConfigStore.load() + + await ConfigStore._testClearOverrides() + #expect(remoteHit) + #expect(!localHit) + #expect(result["remote"] as? Bool == true) + } + + @Test func loadUsesLocalInLocalMode() async { + var localHit = false + var remoteHit = false + await ConfigStore._testSetOverrides(.init( + isRemoteMode: { false }, + loadLocal: { localHit = true; return ["local": true] }, + loadRemote: { remoteHit = true; return ["remote": true] })) + + let result = await ConfigStore.load() + + await ConfigStore._testClearOverrides() + #expect(localHit) + #expect(!remoteHit) + #expect(result["local"] as? Bool == true) + } + + @Test func saveRoutesToRemoteInRemoteMode() async throws { + var localHit = false + var remoteHit = false + await ConfigStore._testSetOverrides(.init( + isRemoteMode: { true }, + saveLocal: { _ in localHit = true }, + saveRemote: { _ in remoteHit = true })) + + try await ConfigStore.save(["remote": true]) + + await ConfigStore._testClearOverrides() + #expect(remoteHit) + #expect(!localHit) + } + + @Test func saveRoutesToLocalInLocalMode() async throws { + var localHit = false + var remoteHit = false + await ConfigStore._testSetOverrides(.init( + isRemoteMode: { false }, + saveLocal: { _ in localHit = true }, + saveRemote: { _ in remoteHit = true })) + + try await ConfigStore.save(["local": true]) + + await ConfigStore._testClearOverrides() + #expect(localHit) + #expect(!remoteHit) + } +} diff --git a/apps/macos/Tests/OpenClawIPCTests/CoverageDumpTests.swift b/apps/macos/Tests/OpenClawIPCTests/CoverageDumpTests.swift new file mode 100644 index 0000000000000000000000000000000000000000..278477448be188383642932428ce972f334e5ef4 --- /dev/null +++ b/apps/macos/Tests/OpenClawIPCTests/CoverageDumpTests.swift @@ -0,0 +1,24 @@ +import Darwin +import Foundation +import Testing + +@Suite(.serialized) +struct CoverageDumpTests { + @Test func periodicallyFlushCoverage() async { + guard ProcessInfo.processInfo.environment["LLVM_PROFILE_FILE"] != nil else { return } + guard let writeProfile = resolveProfileWriteFile() else { return } + let deadline = Date().addingTimeInterval(4) + while Date() < deadline { + _ = writeProfile() + try? await Task.sleep(nanoseconds: 250_000_000) + } + } +} + +private typealias ProfileWriteFn = @convention(c) () -> Int32 + +private func resolveProfileWriteFile() -> ProfileWriteFn? { + let symbol = dlsym(UnsafeMutableRawPointer(bitPattern: -2), "__llvm_profile_write_file") + guard let symbol else { return nil } + return unsafeBitCast(symbol, to: ProfileWriteFn.self) +} diff --git a/apps/macos/Tests/OpenClawIPCTests/CritterIconRendererTests.swift b/apps/macos/Tests/OpenClawIPCTests/CritterIconRendererTests.swift new file mode 100644 index 0000000000000000000000000000000000000000..41baee63e568f860278061ab1768f41cda8f783e --- /dev/null +++ b/apps/macos/Tests/OpenClawIPCTests/CritterIconRendererTests.swift @@ -0,0 +1,37 @@ +import AppKit +import Testing +@testable import OpenClaw + +@Suite +@MainActor +struct CritterIconRendererTests { + @Test func makeIconRendersExpectedSize() { + let image = CritterIconRenderer.makeIcon( + blink: 0.25, + legWiggle: 0.5, + earWiggle: 0.2, + earScale: 1, + earHoles: true, + badge: nil) + + #expect(image.size.width == 18) + #expect(image.size.height == 18) + #expect(image.tiffRepresentation != nil) + } + + @Test func makeIconRendersWithBadge() { + let image = CritterIconRenderer.makeIcon( + blink: 0, + legWiggle: 0, + earWiggle: 0, + earScale: 1, + earHoles: false, + badge: .init(symbolName: "terminal.fill", prominence: .primary)) + + #expect(image.tiffRepresentation != nil) + } + + @Test func critterStatusLabelExercisesHelpers() async { + await CritterStatusLabel.exerciseForTesting() + } +} diff --git a/apps/macos/Tests/OpenClawIPCTests/CronJobEditorSmokeTests.swift b/apps/macos/Tests/OpenClawIPCTests/CronJobEditorSmokeTests.swift new file mode 100644 index 0000000000000000000000000000000000000000..9d833cbe7dd90bfe5b12827da800d317bcaa2176 --- /dev/null +++ b/apps/macos/Tests/OpenClawIPCTests/CronJobEditorSmokeTests.swift @@ -0,0 +1,93 @@ +import SwiftUI +import Testing +@testable import OpenClaw + +@Suite(.serialized) +@MainActor +struct CronJobEditorSmokeTests { + @Test func statusPillBuildsBody() { + _ = StatusPill(text: "ok", tint: .green).body + _ = StatusPill(text: "disabled", tint: .secondary).body + } + + @Test func cronJobEditorBuildsBodyForNewJob() { + let channelsStore = ChannelsStore(isPreview: true) + let view = CronJobEditor( + job: nil, + isSaving: .constant(false), + error: .constant(nil), + channelsStore: channelsStore, + onCancel: {}, + onSave: { _ in }) + _ = view.body + } + + @Test func cronJobEditorBuildsBodyForExistingJob() { + let channelsStore = ChannelsStore(isPreview: true) + let job = CronJob( + id: "job-1", + agentId: "ops", + name: "Daily summary", + description: nil, + enabled: true, + deleteAfterRun: nil, + createdAtMs: 1_700_000_000_000, + updatedAtMs: 1_700_000_000_000, + schedule: .every(everyMs: 3_600_000, anchorMs: 1_700_000_000_000), + sessionTarget: .isolated, + wakeMode: .nextHeartbeat, + payload: .agentTurn( + message: "Summarize the last day", + thinking: "low", + timeoutSeconds: 120, + deliver: true, + channel: "whatsapp", + to: "+15551234567", + bestEffortDeliver: true), + isolation: CronIsolation(postToMainPrefix: "Cron"), + state: CronJobState( + nextRunAtMs: 1_700_000_100_000, + runningAtMs: nil, + lastRunAtMs: 1_700_000_050_000, + lastStatus: "ok", + lastError: nil, + lastDurationMs: 1000)) + + let view = CronJobEditor( + job: job, + isSaving: .constant(false), + error: .constant(nil), + channelsStore: channelsStore, + onCancel: {}, + onSave: { _ in }) + _ = view.body + } + + @Test func cronJobEditorExercisesBuilders() { + let channelsStore = ChannelsStore(isPreview: true) + var view = CronJobEditor( + job: nil, + isSaving: .constant(false), + error: .constant(nil), + channelsStore: channelsStore, + onCancel: {}, + onSave: { _ in }) + view.exerciseForTesting() + } + + @Test func cronJobEditorIncludesDeleteAfterRunForAtSchedule() throws { + let channelsStore = ChannelsStore(isPreview: true) + let view = CronJobEditor( + job: nil, + isSaving: .constant(false), + error: .constant(nil), + channelsStore: channelsStore, + onCancel: {}, + onSave: { _ in }) + + var root: [String: Any] = [:] + view.applyDeleteAfterRun(to: &root, scheduleKind: CronJobEditor.ScheduleKind.at, deleteAfterRun: true) + let raw = root["deleteAfterRun"] as? Bool + #expect(raw == true) + } +} diff --git a/apps/macos/Tests/OpenClawIPCTests/CronModelsTests.swift b/apps/macos/Tests/OpenClawIPCTests/CronModelsTests.swift new file mode 100644 index 0000000000000000000000000000000000000000..f9b5561e81b403c91b6a4158ee2825b32258ec46 --- /dev/null +++ b/apps/macos/Tests/OpenClawIPCTests/CronModelsTests.swift @@ -0,0 +1,129 @@ +import Foundation +import Testing +@testable import OpenClaw + +@Suite +struct CronModelsTests { + @Test func scheduleAtEncodesAndDecodes() throws { + let schedule = CronSchedule.at(atMs: 123) + let data = try JSONEncoder().encode(schedule) + let decoded = try JSONDecoder().decode(CronSchedule.self, from: data) + #expect(decoded == schedule) + } + + @Test func scheduleEveryEncodesAndDecodesWithAnchor() throws { + let schedule = CronSchedule.every(everyMs: 5000, anchorMs: 10000) + let data = try JSONEncoder().encode(schedule) + let decoded = try JSONDecoder().decode(CronSchedule.self, from: data) + #expect(decoded == schedule) + } + + @Test func scheduleCronEncodesAndDecodesWithTimezone() throws { + let schedule = CronSchedule.cron(expr: "*/5 * * * *", tz: "Europe/Vienna") + let data = try JSONEncoder().encode(schedule) + let decoded = try JSONDecoder().decode(CronSchedule.self, from: data) + #expect(decoded == schedule) + } + + @Test func payloadAgentTurnEncodesAndDecodes() throws { + let payload = CronPayload.agentTurn( + message: "hello", + thinking: "low", + timeoutSeconds: 15, + deliver: true, + channel: "whatsapp", + to: "+15551234567", + bestEffortDeliver: false) + let data = try JSONEncoder().encode(payload) + let decoded = try JSONDecoder().decode(CronPayload.self, from: data) + #expect(decoded == payload) + } + + @Test func jobEncodesAndDecodesDeleteAfterRun() throws { + let job = CronJob( + id: "job-1", + agentId: nil, + name: "One-shot", + description: nil, + enabled: true, + deleteAfterRun: true, + createdAtMs: 0, + updatedAtMs: 0, + schedule: .at(atMs: 1_700_000_000_000), + sessionTarget: .main, + wakeMode: .now, + payload: .systemEvent(text: "ping"), + isolation: nil, + state: CronJobState()) + let data = try JSONEncoder().encode(job) + let decoded = try JSONDecoder().decode(CronJob.self, from: data) + #expect(decoded.deleteAfterRun == true) + } + + @Test func scheduleDecodeRejectsUnknownKind() { + let json = """ + {"kind":"wat","atMs":1} + """ + #expect(throws: DecodingError.self) { + _ = try JSONDecoder().decode(CronSchedule.self, from: Data(json.utf8)) + } + } + + @Test func payloadDecodeRejectsUnknownKind() { + let json = """ + {"kind":"wat","text":"hello"} + """ + #expect(throws: DecodingError.self) { + _ = try JSONDecoder().decode(CronPayload.self, from: Data(json.utf8)) + } + } + + @Test func displayNameTrimsWhitespaceAndFallsBack() { + let base = CronJob( + id: "x", + agentId: nil, + name: " hello ", + description: nil, + enabled: true, + deleteAfterRun: nil, + createdAtMs: 0, + updatedAtMs: 0, + schedule: .at(atMs: 0), + sessionTarget: .main, + wakeMode: .now, + payload: .systemEvent(text: "hi"), + isolation: nil, + state: CronJobState()) + #expect(base.displayName == "hello") + + var unnamed = base + unnamed.name = " " + #expect(unnamed.displayName == "Untitled job") + } + + @Test func nextRunDateAndLastRunDateDeriveFromState() { + let job = CronJob( + id: "x", + agentId: nil, + name: "t", + description: nil, + enabled: true, + deleteAfterRun: nil, + createdAtMs: 0, + updatedAtMs: 0, + schedule: .at(atMs: 0), + sessionTarget: .main, + wakeMode: .now, + payload: .systemEvent(text: "hi"), + isolation: nil, + state: CronJobState( + nextRunAtMs: 1_700_000_000_000, + runningAtMs: nil, + lastRunAtMs: 1_700_000_050_000, + lastStatus: nil, + lastError: nil, + lastDurationMs: nil)) + #expect(job.nextRunDate == Date(timeIntervalSince1970: 1_700_000_000)) + #expect(job.lastRunDate == Date(timeIntervalSince1970: 1_700_000_050)) + } +} diff --git a/apps/macos/Tests/OpenClawIPCTests/DeviceModelCatalogTests.swift b/apps/macos/Tests/OpenClawIPCTests/DeviceModelCatalogTests.swift new file mode 100644 index 0000000000000000000000000000000000000000..7d5f1ef679702826277e34c3c4acd1cad5b04e13 --- /dev/null +++ b/apps/macos/Tests/OpenClawIPCTests/DeviceModelCatalogTests.swift @@ -0,0 +1,41 @@ +import Testing +@testable import OpenClaw + +@Suite +struct DeviceModelCatalogTests { + @Test + func symbolPrefersModelIdentifierPrefixes() { + #expect(DeviceModelCatalog + .symbol(deviceFamily: "iPad", modelIdentifier: "iPad16,6", friendlyName: nil) == "ipad") + #expect(DeviceModelCatalog + .symbol(deviceFamily: "iPhone", modelIdentifier: "iPhone17,3", friendlyName: nil) == "iphone") + } + + @Test + func symbolUsesFriendlyNameForMacVariants() { + #expect(DeviceModelCatalog.symbol( + deviceFamily: "Mac", + modelIdentifier: "Mac99,1", + friendlyName: "Mac Studio (2025)") == "macstudio") + #expect(DeviceModelCatalog.symbol( + deviceFamily: "Mac", + modelIdentifier: "Mac99,2", + friendlyName: "Mac mini (2024)") == "macmini") + #expect(DeviceModelCatalog.symbol( + deviceFamily: "Mac", + modelIdentifier: "Mac99,3", + friendlyName: "MacBook Pro (14-inch, 2024)") == "laptopcomputer") + } + + @Test + func symbolFallsBackToDeviceFamily() { + #expect(DeviceModelCatalog.symbol(deviceFamily: "Android", modelIdentifier: "", friendlyName: nil) == "android") + #expect(DeviceModelCatalog.symbol(deviceFamily: "Linux", modelIdentifier: "", friendlyName: nil) == "cpu") + } + + @Test + func presentationUsesBundledModelMappings() { + let presentation = DeviceModelCatalog.presentation(deviceFamily: "iPhone", modelIdentifier: "iPhone1,1") + #expect(presentation?.title == "iPhone") + } +} diff --git a/apps/macos/Tests/OpenClawIPCTests/ExecAllowlistTests.swift b/apps/macos/Tests/OpenClawIPCTests/ExecAllowlistTests.swift new file mode 100644 index 0000000000000000000000000000000000000000..7da886ea794f47cf527fcf991238d546d20c0ea9 --- /dev/null +++ b/apps/macos/Tests/OpenClawIPCTests/ExecAllowlistTests.swift @@ -0,0 +1,49 @@ +import Foundation +import Testing +@testable import OpenClaw + +struct ExecAllowlistTests { + @Test func matchUsesResolvedPath() { + let entry = ExecAllowlistEntry(pattern: "/opt/homebrew/bin/rg") + let resolution = ExecCommandResolution( + rawExecutable: "rg", + resolvedPath: "/opt/homebrew/bin/rg", + executableName: "rg", + cwd: nil) + let match = ExecAllowlistMatcher.match(entries: [entry], resolution: resolution) + #expect(match?.pattern == entry.pattern) + } + + @Test func matchUsesBasenameForSimplePattern() { + let entry = ExecAllowlistEntry(pattern: "rg") + let resolution = ExecCommandResolution( + rawExecutable: "rg", + resolvedPath: "/opt/homebrew/bin/rg", + executableName: "rg", + cwd: nil) + let match = ExecAllowlistMatcher.match(entries: [entry], resolution: resolution) + #expect(match?.pattern == entry.pattern) + } + + @Test func matchIsCaseInsensitive() { + let entry = ExecAllowlistEntry(pattern: "RG") + let resolution = ExecCommandResolution( + rawExecutable: "rg", + resolvedPath: "/opt/homebrew/bin/rg", + executableName: "rg", + cwd: nil) + let match = ExecAllowlistMatcher.match(entries: [entry], resolution: resolution) + #expect(match?.pattern == entry.pattern) + } + + @Test func matchSupportsGlobStar() { + let entry = ExecAllowlistEntry(pattern: "/opt/**/rg") + let resolution = ExecCommandResolution( + rawExecutable: "rg", + resolvedPath: "/opt/homebrew/bin/rg", + executableName: "rg", + cwd: nil) + let match = ExecAllowlistMatcher.match(entries: [entry], resolution: resolution) + #expect(match?.pattern == entry.pattern) + } +} diff --git a/apps/macos/Tests/OpenClawIPCTests/ExecApprovalHelpersTests.swift b/apps/macos/Tests/OpenClawIPCTests/ExecApprovalHelpersTests.swift new file mode 100644 index 0000000000000000000000000000000000000000..760d6c9178e78a66497b2c02711620a9b868502c --- /dev/null +++ b/apps/macos/Tests/OpenClawIPCTests/ExecApprovalHelpersTests.swift @@ -0,0 +1,60 @@ +import Foundation +import Testing +@testable import OpenClaw + +@Suite struct ExecApprovalHelpersTests { + @Test func parseDecisionTrimsAndRejectsInvalid() { + #expect(ExecApprovalHelpers.parseDecision("allow-once") == .allowOnce) + #expect(ExecApprovalHelpers.parseDecision(" allow-always ") == .allowAlways) + #expect(ExecApprovalHelpers.parseDecision("deny") == .deny) + #expect(ExecApprovalHelpers.parseDecision("") == nil) + #expect(ExecApprovalHelpers.parseDecision("nope") == nil) + } + + @Test func allowlistPatternPrefersResolution() { + let resolved = ExecCommandResolution( + rawExecutable: "rg", + resolvedPath: "/opt/homebrew/bin/rg", + executableName: "rg", + cwd: nil) + #expect(ExecApprovalHelpers.allowlistPattern(command: ["rg"], resolution: resolved) == resolved.resolvedPath) + + let rawOnly = ExecCommandResolution( + rawExecutable: "rg", + resolvedPath: nil, + executableName: "rg", + cwd: nil) + #expect(ExecApprovalHelpers.allowlistPattern(command: ["rg"], resolution: rawOnly) == "rg") + #expect(ExecApprovalHelpers.allowlistPattern(command: ["rg"], resolution: nil) == "rg") + #expect(ExecApprovalHelpers.allowlistPattern(command: [], resolution: nil) == nil) + } + + @Test func requiresAskMatchesPolicy() { + let entry = ExecAllowlistEntry(pattern: "/bin/ls", lastUsedAt: nil, lastUsedCommand: nil, lastResolvedPath: nil) + #expect(ExecApprovalHelpers.requiresAsk( + ask: .always, + security: .deny, + allowlistMatch: nil, + skillAllow: false)) + #expect(ExecApprovalHelpers.requiresAsk( + ask: .onMiss, + security: .allowlist, + allowlistMatch: nil, + skillAllow: false)) + #expect(!ExecApprovalHelpers.requiresAsk( + ask: .onMiss, + security: .allowlist, + allowlistMatch: entry, + skillAllow: false)) + #expect(!ExecApprovalHelpers.requiresAsk( + ask: .onMiss, + security: .allowlist, + allowlistMatch: nil, + skillAllow: true)) + #expect(!ExecApprovalHelpers.requiresAsk( + ask: .off, + security: .allowlist, + allowlistMatch: nil, + skillAllow: false)) + } +} diff --git a/apps/macos/Tests/OpenClawIPCTests/ExecApprovalsGatewayPrompterTests.swift b/apps/macos/Tests/OpenClawIPCTests/ExecApprovalsGatewayPrompterTests.swift new file mode 100644 index 0000000000000000000000000000000000000000..4bc75405398e0bfe0c8b3d87b493996238cbd7ee --- /dev/null +++ b/apps/macos/Tests/OpenClawIPCTests/ExecApprovalsGatewayPrompterTests.swift @@ -0,0 +1,56 @@ +import Testing +@testable import OpenClaw + +@Suite +@MainActor +struct ExecApprovalsGatewayPrompterTests { + @Test func sessionMatchPrefersActiveSession() { + let matches = ExecApprovalsGatewayPrompter._testShouldPresent( + mode: .remote, + activeSession: " main ", + requestSession: "main", + lastInputSeconds: nil) + #expect(matches) + + let mismatched = ExecApprovalsGatewayPrompter._testShouldPresent( + mode: .remote, + activeSession: "other", + requestSession: "main", + lastInputSeconds: 0) + #expect(!mismatched) + } + + @Test func sessionFallbackUsesRecentActivity() { + let recent = ExecApprovalsGatewayPrompter._testShouldPresent( + mode: .remote, + activeSession: nil, + requestSession: "main", + lastInputSeconds: 10, + thresholdSeconds: 120) + #expect(recent) + + let stale = ExecApprovalsGatewayPrompter._testShouldPresent( + mode: .remote, + activeSession: nil, + requestSession: "main", + lastInputSeconds: 200, + thresholdSeconds: 120) + #expect(!stale) + } + + @Test func defaultBehaviorMatchesMode() { + let local = ExecApprovalsGatewayPrompter._testShouldPresent( + mode: .local, + activeSession: nil, + requestSession: nil, + lastInputSeconds: 400) + #expect(local) + + let remote = ExecApprovalsGatewayPrompter._testShouldPresent( + mode: .remote, + activeSession: nil, + requestSession: nil, + lastInputSeconds: 400) + #expect(!remote) + } +} diff --git a/apps/macos/Tests/OpenClawIPCTests/FileHandleLegacyAPIGuardTests.swift b/apps/macos/Tests/OpenClawIPCTests/FileHandleLegacyAPIGuardTests.swift new file mode 100644 index 0000000000000000000000000000000000000000..a6836aaa0811ef4cbc54b840caa50d5df8f099e1 --- /dev/null +++ b/apps/macos/Tests/OpenClawIPCTests/FileHandleLegacyAPIGuardTests.swift @@ -0,0 +1,155 @@ +import Foundation +import Testing + +@Suite struct FileHandleLegacyAPIGuardTests { + @Test func sourcesAvoidLegacyNonThrowingFileHandleReadAPIs() throws { + let testFile = URL(fileURLWithPath: #filePath) + let packageRoot = testFile + .deletingLastPathComponent() // OpenClawIPCTests + .deletingLastPathComponent() // Tests + .deletingLastPathComponent() // apps/macos + + let sourcesRoot = packageRoot.appendingPathComponent("Sources") + let swiftFiles = try Self.swiftFiles(under: sourcesRoot) + + var offenders: [String] = [] + for file in swiftFiles { + let raw = try String(contentsOf: file, encoding: .utf8) + let stripped = Self.stripCommentsAndStrings(from: raw) + + if stripped.contains("readDataToEndOfFile(") || stripped.contains(".availableData") { + offenders.append(file.path) + } + } + + if !offenders.isEmpty { + let message = "Found legacy FileHandle reads in:\n" + offenders.joined(separator: "\n") + throw NSError( + domain: "FileHandleLegacyAPIGuardTests", + code: 1, + userInfo: [NSLocalizedDescriptionKey: message]) + } + } + + private static func swiftFiles(under root: URL) throws -> [URL] { + let fm = FileManager() + guard let enumerator = fm.enumerator(at: root, includingPropertiesForKeys: [.isRegularFileKey]) else { + return [] + } + + var files: [URL] = [] + for case let url as URL in enumerator { + guard url.pathExtension == "swift" else { continue } + files.append(url) + } + return files + } + + private static func stripCommentsAndStrings(from source: String) -> String { + enum Mode { + case code + case lineComment + case blockComment(depth: Int) + case string(quoteCount: Int) // 1 = ", 3 = """ + } + + var mode: Mode = .code + var out = "" + out.reserveCapacity(source.count) + + var index = source.startIndex + func peek(_ offset: Int) -> Character? { + guard + let i = source.index(index, offsetBy: offset, limitedBy: source.endIndex), + i < source.endIndex + else { return nil } + return source[i] + } + + while index < source.endIndex { + let ch = source[index] + + switch mode { + case .code: + if ch == "/", peek(1) == "/" { + out.append(" ") + index = source.index(index, offsetBy: 2) + mode = .lineComment + continue + } + if ch == "/", peek(1) == "*" { + out.append(" ") + index = source.index(index, offsetBy: 2) + mode = .blockComment(depth: 1) + continue + } + if ch == "\"" { + let triple = (peek(1) == "\"") && (peek(2) == "\"") + out.append(triple ? " " : " ") + index = source.index(index, offsetBy: triple ? 3 : 1) + mode = .string(quoteCount: triple ? 3 : 1) + continue + } + out.append(ch) + index = source.index(after: index) + + case .lineComment: + if ch == "\n" { + out.append(ch) + index = source.index(after: index) + mode = .code + } else { + out.append(" ") + index = source.index(after: index) + } + + case let .blockComment(depth): + if ch == "/", peek(1) == "*" { + out.append(" ") + index = source.index(index, offsetBy: 2) + mode = .blockComment(depth: depth + 1) + continue + } + if ch == "*", peek(1) == "/" { + out.append(" ") + index = source.index(index, offsetBy: 2) + let newDepth = depth - 1 + mode = newDepth > 0 ? .blockComment(depth: newDepth) : .code + continue + } + out.append(ch == "\n" ? "\n" : " ") + index = source.index(after: index) + + case let .string(quoteCount): + if ch == "\\", quoteCount == 1 { + // Skip escaped character in normal strings. + out.append(" ") + index = source.index(after: index) + if index < source.endIndex { + out.append(" ") + index = source.index(after: index) + } + continue + } + if ch == "\"" { + if quoteCount == 3, peek(1) == "\"", peek(2) == "\"" { + out.append(" ") + index = source.index(index, offsetBy: 3) + mode = .code + continue + } + if quoteCount == 1 { + out.append(" ") + index = source.index(after: index) + mode = .code + continue + } + } + out.append(ch == "\n" ? "\n" : " ") + index = source.index(after: index) + } + } + + return out + } +} diff --git a/apps/macos/Tests/OpenClawIPCTests/FileHandleSafeReadTests.swift b/apps/macos/Tests/OpenClawIPCTests/FileHandleSafeReadTests.swift new file mode 100644 index 0000000000000000000000000000000000000000..3b679a7d5867807880ae6bc9623b3352f7899ecf --- /dev/null +++ b/apps/macos/Tests/OpenClawIPCTests/FileHandleSafeReadTests.swift @@ -0,0 +1,47 @@ +import Foundation +import Testing +@testable import OpenClaw + +@Suite struct FileHandleSafeReadTests { + @Test func readToEndSafelyReturnsEmptyForClosedHandle() { + let pipe = Pipe() + let handle = pipe.fileHandleForReading + try? handle.close() + + let data = handle.readToEndSafely() + #expect(data.isEmpty) + } + + @Test func readSafelyUpToCountReturnsEmptyForClosedHandle() { + let pipe = Pipe() + let handle = pipe.fileHandleForReading + try? handle.close() + + let data = handle.readSafely(upToCount: 16) + #expect(data.isEmpty) + } + + @Test func readToEndSafelyReadsPipeContents() { + let pipe = Pipe() + let writeHandle = pipe.fileHandleForWriting + writeHandle.write(Data("hello".utf8)) + try? writeHandle.close() + + let data = pipe.fileHandleForReading.readToEndSafely() + #expect(String(data: data, encoding: .utf8) == "hello") + } + + @Test func readSafelyUpToCountReadsIncrementally() { + let pipe = Pipe() + let writeHandle = pipe.fileHandleForWriting + writeHandle.write(Data("hello world".utf8)) + try? writeHandle.close() + + let readHandle = pipe.fileHandleForReading + let first = readHandle.readSafely(upToCount: 5) + let second = readHandle.readSafely(upToCount: 32) + + #expect(String(data: first, encoding: .utf8) == "hello") + #expect(String(data: second, encoding: .utf8) == " world") + } +} diff --git a/apps/macos/Tests/OpenClawIPCTests/GatewayAgentChannelTests.swift b/apps/macos/Tests/OpenClawIPCTests/GatewayAgentChannelTests.swift new file mode 100644 index 0000000000000000000000000000000000000000..18972a23bbcb1c0787a54a3bbced533fdfe4c568 --- /dev/null +++ b/apps/macos/Tests/OpenClawIPCTests/GatewayAgentChannelTests.swift @@ -0,0 +1,27 @@ +import Testing +@testable import OpenClaw + +@Suite struct GatewayAgentChannelTests { + @Test func shouldDeliverBlocksWebChat() { + #expect(GatewayAgentChannel.webchat.shouldDeliver(true) == false) + #expect(GatewayAgentChannel.webchat.shouldDeliver(false) == false) + } + + @Test func shouldDeliverAllowsLastAndProviderChannels() { + #expect(GatewayAgentChannel.last.shouldDeliver(true) == true) + #expect(GatewayAgentChannel.whatsapp.shouldDeliver(true) == true) + #expect(GatewayAgentChannel.telegram.shouldDeliver(true) == true) + #expect(GatewayAgentChannel.googlechat.shouldDeliver(true) == true) + #expect(GatewayAgentChannel.bluebubbles.shouldDeliver(true) == true) + #expect(GatewayAgentChannel.last.shouldDeliver(false) == false) + } + + @Test func initRawNormalizesAndFallsBackToLast() { + #expect(GatewayAgentChannel(raw: nil) == .last) + #expect(GatewayAgentChannel(raw: " ") == .last) + #expect(GatewayAgentChannel(raw: "WEBCHAT") == .webchat) + #expect(GatewayAgentChannel(raw: "googlechat") == .googlechat) + #expect(GatewayAgentChannel(raw: "BLUEBUBBLES") == .bluebubbles) + #expect(GatewayAgentChannel(raw: "unknown") == .last) + } +} diff --git a/apps/macos/Tests/OpenClawIPCTests/GatewayAutostartPolicyTests.swift b/apps/macos/Tests/OpenClawIPCTests/GatewayAutostartPolicyTests.swift new file mode 100644 index 0000000000000000000000000000000000000000..f2fea5fc458a469eb2ac04ac7f7d9fa6752c08ad --- /dev/null +++ b/apps/macos/Tests/OpenClawIPCTests/GatewayAutostartPolicyTests.swift @@ -0,0 +1,24 @@ +import Testing +@testable import OpenClaw + +@Suite(.serialized) +struct GatewayAutostartPolicyTests { + @Test func startsGatewayOnlyWhenLocalAndNotPaused() { + #expect(GatewayAutostartPolicy.shouldStartGateway(mode: .local, paused: false)) + #expect(!GatewayAutostartPolicy.shouldStartGateway(mode: .local, paused: true)) + #expect(!GatewayAutostartPolicy.shouldStartGateway(mode: .remote, paused: false)) + #expect(!GatewayAutostartPolicy.shouldStartGateway(mode: .unconfigured, paused: false)) + } + + @Test func ensuresLaunchAgentWhenLocalAndNotAttachOnly() { + #expect(GatewayAutostartPolicy.shouldEnsureLaunchAgent( + mode: .local, + paused: false)) + #expect(!GatewayAutostartPolicy.shouldEnsureLaunchAgent( + mode: .local, + paused: true)) + #expect(!GatewayAutostartPolicy.shouldEnsureLaunchAgent( + mode: .remote, + paused: false)) + } +} diff --git a/apps/macos/Tests/OpenClawIPCTests/GatewayChannelConfigureTests.swift b/apps/macos/Tests/OpenClawIPCTests/GatewayChannelConfigureTests.swift new file mode 100644 index 0000000000000000000000000000000000000000..7200af03cddea690c0e719a411cb52236d8f3e2d --- /dev/null +++ b/apps/macos/Tests/OpenClawIPCTests/GatewayChannelConfigureTests.swift @@ -0,0 +1,286 @@ +import OpenClawKit +import Foundation +import os +import Testing +@testable import OpenClaw + +@Suite struct GatewayConnectionTests { + private final class FakeWebSocketTask: WebSocketTasking, @unchecked Sendable { + private let connectRequestID = OSAllocatedUnfairLock(initialState: nil) + private let pendingReceiveHandler = + OSAllocatedUnfairLock<(@Sendable (Result) + -> Void)?>(initialState: nil) + private let cancelCount = OSAllocatedUnfairLock(initialState: 0) + private let sendCount = OSAllocatedUnfairLock(initialState: 0) + private let helloDelayMs: Int + + var state: URLSessionTask.State = .suspended + + init(helloDelayMs: Int = 0) { + self.helloDelayMs = helloDelayMs + } + + func snapshotCancelCount() -> Int { self.cancelCount.withLock { $0 } } + + func resume() { + self.state = .running + } + + func cancel(with closeCode: URLSessionWebSocketTask.CloseCode, reason: Data?) { + _ = (closeCode, reason) + self.state = .canceling + self.cancelCount.withLock { $0 += 1 } + let handler = self.pendingReceiveHandler.withLock { handler in + defer { handler = nil } + return handler + } + handler?(Result.failure(URLError(.cancelled))) + } + + func send(_ message: URLSessionWebSocketTask.Message) async throws { + let currentSendCount = self.sendCount.withLock { count in + defer { count += 1 } + return count + } + + // First send is the connect handshake request. Subsequent sends are request frames. + if currentSendCount == 0 { + guard case let .data(data) = message else { return } + if let obj = try? JSONSerialization.jsonObject(with: data) as? [String: Any], + (obj["type"] as? String) == "req", + (obj["method"] as? String) == "connect", + let id = obj["id"] as? String + { + self.connectRequestID.withLock { $0 = id } + } + return + } + + guard case let .data(data) = message else { return } + guard + let obj = try? JSONSerialization.jsonObject(with: data) as? [String: Any], + (obj["type"] as? String) == "req", + let id = obj["id"] as? String + else { + return + } + + let response = Self.responseData(id: id) + let handler = self.pendingReceiveHandler.withLock { $0 } + handler?(Result.success(.data(response))) + } + + func receive() async throws -> URLSessionWebSocketTask.Message { + if self.helloDelayMs > 0 { + try await Task.sleep(nanoseconds: UInt64(self.helloDelayMs) * 1_000_000) + } + let id = self.connectRequestID.withLock { $0 } ?? "connect" + return .data(Self.connectOkData(id: id)) + } + + func receive( + completionHandler: @escaping @Sendable (Result) -> Void) + { + self.pendingReceiveHandler.withLock { $0 = completionHandler } + } + + func emitIncoming(_ data: Data) { + let handler = self.pendingReceiveHandler.withLock { $0 } + handler?(Result.success(.data(data))) + } + + private static func connectOkData(id: String) -> Data { + let json = """ + { + "type": "res", + "id": "\(id)", + "ok": true, + "payload": { + "type": "hello-ok", + "protocol": 2, + "server": { "version": "test", "connId": "test" }, + "features": { "methods": [], "events": [] }, + "snapshot": { + "presence": [ { "ts": 1 } ], + "health": {}, + "stateVersion": { "presence": 0, "health": 0 }, + "uptimeMs": 0 + }, + "policy": { "maxPayload": 1, "maxBufferedBytes": 1, "tickIntervalMs": 30000 } + } + } + """ + return Data(json.utf8) + } + + private static func responseData(id: String) -> Data { + let json = """ + { + "type": "res", + "id": "\(id)", + "ok": true, + "payload": { "ok": true } + } + """ + return Data(json.utf8) + } + } + + private final class FakeWebSocketSession: WebSocketSessioning, @unchecked Sendable { + private let makeCount = OSAllocatedUnfairLock(initialState: 0) + private let tasks = OSAllocatedUnfairLock(initialState: [FakeWebSocketTask]()) + private let helloDelayMs: Int + + init(helloDelayMs: Int = 0) { + self.helloDelayMs = helloDelayMs + } + + func snapshotMakeCount() -> Int { self.makeCount.withLock { $0 } } + func snapshotCancelCount() -> Int { + self.tasks.withLock { tasks in + tasks.reduce(0) { $0 + $1.snapshotCancelCount() } + } + } + + func latestTask() -> FakeWebSocketTask? { + self.tasks.withLock { $0.last } + } + + func makeWebSocketTask(url: URL) -> WebSocketTaskBox { + _ = url + self.makeCount.withLock { $0 += 1 } + let task = FakeWebSocketTask(helloDelayMs: self.helloDelayMs) + self.tasks.withLock { $0.append(task) } + return WebSocketTaskBox(task: task) + } + } + + private final class ConfigSource: @unchecked Sendable { + private let token = OSAllocatedUnfairLock(initialState: nil) + + init(token: String?) { + self.token.withLock { $0 = token } + } + + func snapshotToken() -> String? { self.token.withLock { $0 } } + func setToken(_ value: String?) { self.token.withLock { $0 = value } } + } + + @Test func requestReusesSingleWebSocketForSameConfig() async throws { + let session = FakeWebSocketSession() + let url = URL(string: "ws://example.invalid")! + let cfg = ConfigSource(token: nil) + let conn = GatewayConnection( + configProvider: { (url: url, token: cfg.snapshotToken(), password: nil) }, + sessionBox: WebSocketSessionBox(session: session)) + + _ = try await conn.request(method: "status", params: nil) + #expect(session.snapshotMakeCount() == 1) + + _ = try await conn.request(method: "status", params: nil) + #expect(session.snapshotMakeCount() == 1) + #expect(session.snapshotCancelCount() == 0) + } + + @Test func requestReconfiguresAndCancelsOnTokenChange() async throws { + let session = FakeWebSocketSession() + let url = URL(string: "ws://example.invalid")! + let cfg = ConfigSource(token: "a") + let conn = GatewayConnection( + configProvider: { (url: url, token: cfg.snapshotToken(), password: nil) }, + sessionBox: WebSocketSessionBox(session: session)) + + _ = try await conn.request(method: "status", params: nil) + #expect(session.snapshotMakeCount() == 1) + + cfg.setToken("b") + _ = try await conn.request(method: "status", params: nil) + #expect(session.snapshotMakeCount() == 2) + #expect(session.snapshotCancelCount() == 1) + } + + @Test func concurrentRequestsStillUseSingleWebSocket() async throws { + let session = FakeWebSocketSession(helloDelayMs: 150) + let url = URL(string: "ws://example.invalid")! + let cfg = ConfigSource(token: nil) + let conn = GatewayConnection( + configProvider: { (url: url, token: cfg.snapshotToken(), password: nil) }, + sessionBox: WebSocketSessionBox(session: session)) + + async let r1: Data = conn.request(method: "status", params: nil) + async let r2: Data = conn.request(method: "status", params: nil) + _ = try await (r1, r2) + + #expect(session.snapshotMakeCount() == 1) + } + + @Test func subscribeReplaysLatestSnapshot() async throws { + let session = FakeWebSocketSession() + let url = URL(string: "ws://example.invalid")! + let cfg = ConfigSource(token: nil) + let conn = GatewayConnection( + configProvider: { (url: url, token: cfg.snapshotToken(), password: nil) }, + sessionBox: WebSocketSessionBox(session: session)) + + _ = try await conn.request(method: "status", params: nil) + + let stream = await conn.subscribe(bufferingNewest: 10) + var iterator = stream.makeAsyncIterator() + let first = await iterator.next() + + guard case let .snapshot(snap) = first else { + Issue.record("expected snapshot, got \(String(describing: first))") + return + } + #expect(snap.type == "hello-ok") + } + + @Test func subscribeEmitsSeqGapBeforeEvent() async throws { + let session = FakeWebSocketSession() + let url = URL(string: "ws://example.invalid")! + let cfg = ConfigSource(token: nil) + let conn = GatewayConnection( + configProvider: { (url: url, token: cfg.snapshotToken(), password: nil) }, + sessionBox: WebSocketSessionBox(session: session)) + + let stream = await conn.subscribe(bufferingNewest: 10) + var iterator = stream.makeAsyncIterator() + + _ = try await conn.request(method: "status", params: nil) + _ = await iterator.next() // snapshot + + let evt1 = Data( + """ + {"type":"event","event":"presence","payload":{"presence":[]},"seq":1} + """.utf8) + session.latestTask()?.emitIncoming(evt1) + + let firstEvent = await iterator.next() + guard case let .event(firstFrame) = firstEvent else { + Issue.record("expected event, got \(String(describing: firstEvent))") + return + } + #expect(firstFrame.seq == 1) + + let evt3 = Data( + """ + {"type":"event","event":"presence","payload":{"presence":[]},"seq":3} + """.utf8) + session.latestTask()?.emitIncoming(evt3) + + let gap = await iterator.next() + guard case let .seqGap(expected, received) = gap else { + Issue.record("expected seqGap, got \(String(describing: gap))") + return + } + #expect(expected == 2) + #expect(received == 3) + + let secondEvent = await iterator.next() + guard case let .event(secondFrame) = secondEvent else { + Issue.record("expected event, got \(String(describing: secondEvent))") + return + } + #expect(secondFrame.seq == 3) + } +} diff --git a/apps/macos/Tests/OpenClawIPCTests/GatewayChannelConnectTests.swift b/apps/macos/Tests/OpenClawIPCTests/GatewayChannelConnectTests.swift new file mode 100644 index 0000000000000000000000000000000000000000..bda06e9cf56d15366d1f2251de8eab4975aab4c0 --- /dev/null +++ b/apps/macos/Tests/OpenClawIPCTests/GatewayChannelConnectTests.swift @@ -0,0 +1,160 @@ +import OpenClawKit +import Foundation +import os +import Testing +@testable import OpenClaw + +@Suite struct GatewayChannelConnectTests { + private enum FakeResponse { + case helloOk(delayMs: Int) + case invalid(delayMs: Int) + } + + private final class FakeWebSocketTask: WebSocketTasking, @unchecked Sendable { + private let response: FakeResponse + private let connectRequestID = OSAllocatedUnfairLock(initialState: nil) + private let pendingReceiveHandler = + OSAllocatedUnfairLock<(@Sendable (Result) -> Void)?>( + initialState: nil) + + var state: URLSessionTask.State = .suspended + + init(response: FakeResponse) { + self.response = response + } + + func resume() { + self.state = .running + } + + func cancel(with closeCode: URLSessionWebSocketTask.CloseCode, reason: Data?) { + _ = (closeCode, reason) + self.state = .canceling + let handler = self.pendingReceiveHandler.withLock { handler in + defer { handler = nil } + return handler + } + handler?(Result.failure(URLError(.cancelled))) + } + + func send(_ message: URLSessionWebSocketTask.Message) async throws { + let data: Data? = switch message { + case let .data(d): d + case let .string(s): s.data(using: .utf8) + @unknown default: nil + } + guard let data else { return } + if let obj = try? JSONSerialization.jsonObject(with: data) as? [String: Any], + obj["type"] as? String == "req", + obj["method"] as? String == "connect", + let id = obj["id"] as? String + { + self.connectRequestID.withLock { $0 = id } + } + } + + func receive() async throws -> URLSessionWebSocketTask.Message { + let delayMs: Int + let msg: URLSessionWebSocketTask.Message + switch self.response { + case let .helloOk(ms): + delayMs = ms + let id = self.connectRequestID.withLock { $0 } ?? "connect" + msg = .data(Self.connectOkData(id: id)) + case let .invalid(ms): + delayMs = ms + msg = .string("not json") + } + try await Task.sleep(nanoseconds: UInt64(delayMs) * 1_000_000) + return msg + } + + func receive( + completionHandler: @escaping @Sendable (Result) -> Void) + { + // The production channel sets up a continuous receive loop after hello. + // Tests only need the handshake receive; keep the loop idle. + self.pendingReceiveHandler.withLock { $0 = completionHandler } + } + + private static func connectOkData(id: String) -> Data { + let json = """ + { + "type": "res", + "id": "\(id)", + "ok": true, + "payload": { + "type": "hello-ok", + "protocol": 2, + "server": { "version": "test", "connId": "test" }, + "features": { "methods": [], "events": [] }, + "snapshot": { + "presence": [ { "ts": 1 } ], + "health": {}, + "stateVersion": { "presence": 0, "health": 0 }, + "uptimeMs": 0 + }, + "policy": { "maxPayload": 1, "maxBufferedBytes": 1, "tickIntervalMs": 30000 } + } + } + """ + return Data(json.utf8) + } + } + + private final class FakeWebSocketSession: WebSocketSessioning, @unchecked Sendable { + private let response: FakeResponse + private let makeCount = OSAllocatedUnfairLock(initialState: 0) + + init(response: FakeResponse) { + self.response = response + } + + func snapshotMakeCount() -> Int { self.makeCount.withLock { $0 } } + + func makeWebSocketTask(url: URL) -> WebSocketTaskBox { + _ = url + self.makeCount.withLock { $0 += 1 } + let task = FakeWebSocketTask(response: self.response) + return WebSocketTaskBox(task: task) + } + } + + @Test func concurrentConnectIsSingleFlightOnSuccess() async throws { + let session = FakeWebSocketSession(response: .helloOk(delayMs: 200)) + let channel = GatewayChannelActor( + url: URL(string: "ws://example.invalid")!, + token: nil, + session: WebSocketSessionBox(session: session)) + + let t1 = Task { try await channel.connect() } + let t2 = Task { try await channel.connect() } + + _ = try await t1.value + _ = try await t2.value + + #expect(session.snapshotMakeCount() == 1) + } + + @Test func concurrentConnectSharesFailure() async { + let session = FakeWebSocketSession(response: .invalid(delayMs: 200)) + let channel = GatewayChannelActor( + url: URL(string: "ws://example.invalid")!, + token: nil, + session: WebSocketSessionBox(session: session)) + + let t1 = Task { try await channel.connect() } + let t2 = Task { try await channel.connect() } + + let r1 = await t1.result + let r2 = await t2.result + + #expect({ + if case .failure = r1 { true } else { false } + }()) + #expect({ + if case .failure = r2 { true } else { false } + }()) + #expect(session.snapshotMakeCount() == 1) + } +} diff --git a/apps/macos/Tests/OpenClawIPCTests/GatewayChannelRequestTests.swift b/apps/macos/Tests/OpenClawIPCTests/GatewayChannelRequestTests.swift new file mode 100644 index 0000000000000000000000000000000000000000..94edb6ebf77ff916f99cf09d69aabc1be2b78c82 --- /dev/null +++ b/apps/macos/Tests/OpenClawIPCTests/GatewayChannelRequestTests.swift @@ -0,0 +1,134 @@ +import OpenClawKit +import Foundation +import os +import Testing +@testable import OpenClaw + +@Suite struct GatewayChannelRequestTests { + private final class FakeWebSocketTask: WebSocketTasking, @unchecked Sendable { + private let requestSendDelayMs: Int + private let connectRequestID = OSAllocatedUnfairLock(initialState: nil) + private let pendingReceiveHandler = + OSAllocatedUnfairLock<(@Sendable (Result) + -> Void)?>(initialState: nil) + private let sendCount = OSAllocatedUnfairLock(initialState: 0) + + var state: URLSessionTask.State = .suspended + + init(requestSendDelayMs: Int) { + self.requestSendDelayMs = requestSendDelayMs + } + + func resume() { + self.state = .running + } + + func cancel(with closeCode: URLSessionWebSocketTask.CloseCode, reason: Data?) { + _ = (closeCode, reason) + self.state = .canceling + let handler = self.pendingReceiveHandler.withLock { handler in + defer { handler = nil } + return handler + } + handler?(Result.failure(URLError(.cancelled))) + } + + func send(_ message: URLSessionWebSocketTask.Message) async throws { + _ = message + let currentSendCount = self.sendCount.withLock { count in + defer { count += 1 } + return count + } + + // First send is the connect handshake. Second send is the request frame. + if currentSendCount == 0 { + let data: Data? = switch message { + case let .data(d): d + case let .string(s): s.data(using: .utf8) + @unknown default: nil + } + guard let data else { return } + if let obj = try? JSONSerialization.jsonObject(with: data) as? [String: Any], + obj["type"] as? String == "req", + obj["method"] as? String == "connect", + let id = obj["id"] as? String + { + self.connectRequestID.withLock { $0 = id } + } + } + if currentSendCount == 1 { + try await Task.sleep(nanoseconds: UInt64(self.requestSendDelayMs) * 1_000_000) + throw URLError(.cannotConnectToHost) + } + } + + func receive() async throws -> URLSessionWebSocketTask.Message { + let id = self.connectRequestID.withLock { $0 } ?? "connect" + return .data(Self.connectOkData(id: id)) + } + + func receive( + completionHandler: @escaping @Sendable (Result) -> Void) + { + self.pendingReceiveHandler.withLock { $0 = completionHandler } + } + + private static func connectOkData(id: String) -> Data { + let json = """ + { + "type": "res", + "id": "\(id)", + "ok": true, + "payload": { + "type": "hello-ok", + "protocol": 2, + "server": { "version": "test", "connId": "test" }, + "features": { "methods": [], "events": [] }, + "snapshot": { + "presence": [ { "ts": 1 } ], + "health": {}, + "stateVersion": { "presence": 0, "health": 0 }, + "uptimeMs": 0 + }, + "policy": { "maxPayload": 1, "maxBufferedBytes": 1, "tickIntervalMs": 30000 } + } + } + """ + return Data(json.utf8) + } + } + + private final class FakeWebSocketSession: WebSocketSessioning, @unchecked Sendable { + private let requestSendDelayMs: Int + + init(requestSendDelayMs: Int) { + self.requestSendDelayMs = requestSendDelayMs + } + + func makeWebSocketTask(url: URL) -> WebSocketTaskBox { + _ = url + let task = FakeWebSocketTask(requestSendDelayMs: self.requestSendDelayMs) + return WebSocketTaskBox(task: task) + } + } + + @Test func requestTimeoutThenSendFailureDoesNotDoubleResume() async { + let session = FakeWebSocketSession(requestSendDelayMs: 100) + let channel = GatewayChannelActor( + url: URL(string: "ws://example.invalid")!, + token: nil, + session: WebSocketSessionBox(session: session)) + + do { + _ = try await channel.request(method: "test", params: nil, timeoutMs: 10) + Issue.record("Expected request to time out") + } catch { + let ns = error as NSError + #expect(ns.domain == "Gateway") + #expect(ns.code == 5) + } + + // Give the delayed send failure task time to run; this used to crash due to a double-resume. + try? await Task.sleep(nanoseconds: 250 * 1_000_000) + } +} diff --git a/apps/macos/Tests/OpenClawIPCTests/GatewayChannelShutdownTests.swift b/apps/macos/Tests/OpenClawIPCTests/GatewayChannelShutdownTests.swift new file mode 100644 index 0000000000000000000000000000000000000000..eea7774adf2e18860f2207717f9200f817223d73 --- /dev/null +++ b/apps/macos/Tests/OpenClawIPCTests/GatewayChannelShutdownTests.swift @@ -0,0 +1,129 @@ +import OpenClawKit +import Foundation +import os +import Testing +@testable import OpenClaw + +@Suite struct GatewayChannelShutdownTests { + private final class FakeWebSocketTask: WebSocketTasking, @unchecked Sendable { + private let connectRequestID = OSAllocatedUnfairLock(initialState: nil) + private let pendingReceiveHandler = + OSAllocatedUnfairLock<(@Sendable (Result) + -> Void)?>(initialState: nil) + private let cancelCount = OSAllocatedUnfairLock(initialState: 0) + + var state: URLSessionTask.State = .suspended + + func snapshotCancelCount() -> Int { self.cancelCount.withLock { $0 } } + + func resume() { + self.state = .running + } + + func cancel(with closeCode: URLSessionWebSocketTask.CloseCode, reason: Data?) { + _ = (closeCode, reason) + self.state = .canceling + self.cancelCount.withLock { $0 += 1 } + let handler = self.pendingReceiveHandler.withLock { handler in + defer { handler = nil } + return handler + } + handler?(Result.failure(URLError(.cancelled))) + } + + func send(_ message: URLSessionWebSocketTask.Message) async throws { + let data: Data? = switch message { + case let .data(d): d + case let .string(s): s.data(using: .utf8) + @unknown default: nil + } + guard let data else { return } + if let obj = try? JSONSerialization.jsonObject(with: data) as? [String: Any], + obj["type"] as? String == "req", + obj["method"] as? String == "connect", + let id = obj["id"] as? String + { + self.connectRequestID.withLock { $0 = id } + } + } + + func receive() async throws -> URLSessionWebSocketTask.Message { + let id = self.connectRequestID.withLock { $0 } ?? "connect" + return .data(Self.connectOkData(id: id)) + } + + func receive( + completionHandler: @escaping @Sendable (Result) -> Void) + { + self.pendingReceiveHandler.withLock { $0 = completionHandler } + } + + func triggerReceiveFailure() { + let handler = self.pendingReceiveHandler.withLock { $0 } + handler?(Result.failure(URLError(.networkConnectionLost))) + } + + private static func connectOkData(id: String) -> Data { + let json = """ + { + "type": "res", + "id": "\(id)", + "ok": true, + "payload": { + "type": "hello-ok", + "protocol": 2, + "server": { "version": "test", "connId": "test" }, + "features": { "methods": [], "events": [] }, + "snapshot": { + "presence": [ { "ts": 1 } ], + "health": {}, + "stateVersion": { "presence": 0, "health": 0 }, + "uptimeMs": 0 + }, + "policy": { "maxPayload": 1, "maxBufferedBytes": 1, "tickIntervalMs": 30000 } + } + } + """ + return Data(json.utf8) + } + } + + private final class FakeWebSocketSession: WebSocketSessioning, @unchecked Sendable { + private let makeCount = OSAllocatedUnfairLock(initialState: 0) + private let tasks = OSAllocatedUnfairLock(initialState: [FakeWebSocketTask]()) + + func snapshotMakeCount() -> Int { self.makeCount.withLock { $0 } } + func latestTask() -> FakeWebSocketTask? { self.tasks.withLock { $0.last } } + + func makeWebSocketTask(url: URL) -> WebSocketTaskBox { + _ = url + self.makeCount.withLock { $0 += 1 } + let task = FakeWebSocketTask() + self.tasks.withLock { $0.append(task) } + return WebSocketTaskBox(task: task) + } + } + + @Test func shutdownPreventsReconnectLoopFromReceiveFailure() async throws { + let session = FakeWebSocketSession() + let channel = GatewayChannelActor( + url: URL(string: "ws://example.invalid")!, + token: nil, + session: WebSocketSessionBox(session: session)) + + // Establish a connection so `listen()` is active. + try await channel.connect() + #expect(session.snapshotMakeCount() == 1) + + // Simulate a socket receive failure, which would normally schedule a reconnect. + session.latestTask()?.triggerReceiveFailure() + + // Shut down quickly, before backoff reconnect triggers. + await channel.shutdown() + + // Wait longer than the default reconnect backoff (500ms) to ensure no reconnect happens. + try? await Task.sleep(nanoseconds: 750 * 1_000_000) + + #expect(session.snapshotMakeCount() == 1) + } +} diff --git a/apps/macos/Tests/OpenClawIPCTests/GatewayConnectionControlTests.swift b/apps/macos/Tests/OpenClawIPCTests/GatewayConnectionControlTests.swift new file mode 100644 index 0000000000000000000000000000000000000000..e95cf7a282ddc7cb41137d9c3e2a3c8a95bb03e9 --- /dev/null +++ b/apps/macos/Tests/OpenClawIPCTests/GatewayConnectionControlTests.swift @@ -0,0 +1,59 @@ +import OpenClawKit +import Foundation +import Testing +@testable import OpenClaw +@testable import OpenClawIPC + +private final class FakeWebSocketTask: WebSocketTasking, @unchecked Sendable { + var state: URLSessionTask.State = .running + + func resume() {} + + func cancel(with _: URLSessionWebSocketTask.CloseCode, reason _: Data?) { + self.state = .canceling + } + + func send(_: URLSessionWebSocketTask.Message) async throws {} + + func receive() async throws -> URLSessionWebSocketTask.Message { + throw URLError(.cannotConnectToHost) + } + + func receive(completionHandler: @escaping @Sendable (Result) -> Void) { + completionHandler(.failure(URLError(.cannotConnectToHost))) + } +} + +private final class FakeWebSocketSession: WebSocketSessioning, @unchecked Sendable { + func makeWebSocketTask(url _: URL) -> WebSocketTaskBox { + WebSocketTaskBox(task: FakeWebSocketTask()) + } +} + +private func makeTestGatewayConnection() -> GatewayConnection { + GatewayConnection( + configProvider: { + (url: URL(string: "ws://127.0.0.1:1")!, token: nil, password: nil) + }, + sessionBox: WebSocketSessionBox(session: FakeWebSocketSession())) +} + +@Suite(.serialized) struct GatewayConnectionControlTests { + @Test func statusFailsWhenProcessMissing() async { + let connection = makeTestGatewayConnection() + let result = await connection.status() + #expect(result.ok == false) + #expect(result.error != nil) + } + + @Test func rejectEmptyMessage() async { + let connection = makeTestGatewayConnection() + let result = await connection.sendAgent( + message: "", + thinking: nil, + sessionKey: "main", + deliver: false, + to: nil) + #expect(result.ok == false) + } +} diff --git a/apps/macos/Tests/OpenClawIPCTests/GatewayDiscoveryModelTests.swift b/apps/macos/Tests/OpenClawIPCTests/GatewayDiscoveryModelTests.swift new file mode 100644 index 0000000000000000000000000000000000000000..02888c73870946504be6f36b7d0fadb1bcaf1887 --- /dev/null +++ b/apps/macos/Tests/OpenClawIPCTests/GatewayDiscoveryModelTests.swift @@ -0,0 +1,124 @@ +import OpenClawDiscovery +import Testing + +@Suite +@MainActor +struct GatewayDiscoveryModelTests { + @Test func localGatewayMatchesLanHost() { + let local = GatewayDiscoveryModel.LocalIdentity( + hostTokens: ["studio"], + displayTokens: []) + #expect(GatewayDiscoveryModel.isLocalGateway( + lanHost: "studio.local", + tailnetDns: nil, + displayName: nil, + serviceName: nil, + local: local)) + } + + @Test func localGatewayMatchesTailnetDns() { + let local = GatewayDiscoveryModel.LocalIdentity( + hostTokens: ["studio"], + displayTokens: []) + #expect(GatewayDiscoveryModel.isLocalGateway( + lanHost: nil, + tailnetDns: "studio.tailnet.example", + displayName: nil, + serviceName: nil, + local: local)) + } + + @Test func localGatewayMatchesDisplayName() { + let local = GatewayDiscoveryModel.LocalIdentity( + hostTokens: [], + displayTokens: ["peter's mac studio"]) + #expect(GatewayDiscoveryModel.isLocalGateway( + lanHost: nil, + tailnetDns: nil, + displayName: "Peter's Mac Studio (OpenClaw)", + serviceName: nil, + local: local)) + } + + @Test func remoteGatewayDoesNotMatch() { + let local = GatewayDiscoveryModel.LocalIdentity( + hostTokens: ["studio"], + displayTokens: ["peter's mac studio"]) + #expect(!GatewayDiscoveryModel.isLocalGateway( + lanHost: "other.local", + tailnetDns: "other.tailnet.example", + displayName: "Other Mac", + serviceName: "other-gateway", + local: local)) + } + + @Test func localGatewayMatchesServiceName() { + let local = GatewayDiscoveryModel.LocalIdentity( + hostTokens: ["studio"], + displayTokens: []) + #expect(GatewayDiscoveryModel.isLocalGateway( + lanHost: nil, + tailnetDns: nil, + displayName: nil, + serviceName: "studio-gateway", + local: local)) + } + + @Test func serviceNameDoesNotFalsePositiveOnSubstringHostToken() { + let local = GatewayDiscoveryModel.LocalIdentity( + hostTokens: ["steipete"], + displayTokens: []) + #expect(!GatewayDiscoveryModel.isLocalGateway( + lanHost: nil, + tailnetDns: nil, + displayName: nil, + serviceName: "steipetacstudio (OpenClaw)", + local: local)) + #expect(GatewayDiscoveryModel.isLocalGateway( + lanHost: nil, + tailnetDns: nil, + displayName: nil, + serviceName: "steipete (OpenClaw)", + local: local)) + } + + @Test func parsesGatewayTXTFields() { + let parsed = GatewayDiscoveryModel.parseGatewayTXT([ + "lanHost": " studio.local ", + "tailnetDns": " peters-mac-studio-1.ts.net ", + "sshPort": " 2222 ", + "gatewayPort": " 18799 ", + "cliPath": " /opt/openclaw ", + ]) + #expect(parsed.lanHost == "studio.local") + #expect(parsed.tailnetDns == "peters-mac-studio-1.ts.net") + #expect(parsed.sshPort == 2222) + #expect(parsed.gatewayPort == 18799) + #expect(parsed.cliPath == "/opt/openclaw") + } + + @Test func parsesGatewayTXTDefaults() { + let parsed = GatewayDiscoveryModel.parseGatewayTXT([ + "lanHost": " ", + "tailnetDns": "\n", + "gatewayPort": "nope", + "sshPort": "nope", + ]) + #expect(parsed.lanHost == nil) + #expect(parsed.tailnetDns == nil) + #expect(parsed.sshPort == 22) + #expect(parsed.gatewayPort == nil) + #expect(parsed.cliPath == nil) + } + + @Test func buildsSSHTarget() { + #expect(GatewayDiscoveryModel.buildSSHTarget( + user: "peter", + host: "studio.local", + port: 22) == "peter@studio.local") + #expect(GatewayDiscoveryModel.buildSSHTarget( + user: "peter", + host: "studio.local", + port: 2201) == "peter@studio.local:2201") + } +} diff --git a/apps/macos/Tests/OpenClawIPCTests/GatewayEndpointStoreTests.swift b/apps/macos/Tests/OpenClawIPCTests/GatewayEndpointStoreTests.swift new file mode 100644 index 0000000000000000000000000000000000000000..8ab50b6535faf7a2fbc26e165bc321949521dd53 --- /dev/null +++ b/apps/macos/Tests/OpenClawIPCTests/GatewayEndpointStoreTests.swift @@ -0,0 +1,184 @@ +import Foundation +import Testing +@testable import OpenClaw + +@Suite struct GatewayEndpointStoreTests { + private func makeDefaults() -> UserDefaults { + let suiteName = "GatewayEndpointStoreTests.\(UUID().uuidString)" + let defaults = UserDefaults(suiteName: suiteName)! + defaults.removePersistentDomain(forName: suiteName) + return defaults + } + + @Test func resolveGatewayTokenPrefersEnvAndFallsBackToLaunchd() { + let snapshot = LaunchAgentPlistSnapshot( + programArguments: [], + environment: ["OPENCLAW_GATEWAY_TOKEN": "launchd-token"], + stdoutPath: nil, + stderrPath: nil, + port: nil, + bind: nil, + token: "launchd-token", + password: nil) + + let envToken = GatewayEndpointStore._testResolveGatewayToken( + isRemote: false, + root: [:], + env: ["OPENCLAW_GATEWAY_TOKEN": "env-token"], + launchdSnapshot: snapshot) + #expect(envToken == "env-token") + + let fallbackToken = GatewayEndpointStore._testResolveGatewayToken( + isRemote: false, + root: [:], + env: [:], + launchdSnapshot: snapshot) + #expect(fallbackToken == "launchd-token") + } + + @Test func resolveGatewayTokenIgnoresLaunchdInRemoteMode() { + let snapshot = LaunchAgentPlistSnapshot( + programArguments: [], + environment: ["OPENCLAW_GATEWAY_TOKEN": "launchd-token"], + stdoutPath: nil, + stderrPath: nil, + port: nil, + bind: nil, + token: "launchd-token", + password: nil) + + let token = GatewayEndpointStore._testResolveGatewayToken( + isRemote: true, + root: [:], + env: [:], + launchdSnapshot: snapshot) + #expect(token == nil) + } + + @Test func resolveGatewayPasswordFallsBackToLaunchd() { + let snapshot = LaunchAgentPlistSnapshot( + programArguments: [], + environment: ["OPENCLAW_GATEWAY_PASSWORD": "launchd-pass"], + stdoutPath: nil, + stderrPath: nil, + port: nil, + bind: nil, + token: nil, + password: "launchd-pass") + + let password = GatewayEndpointStore._testResolveGatewayPassword( + isRemote: false, + root: [:], + env: [:], + launchdSnapshot: snapshot) + #expect(password == "launchd-pass") + } + + @Test func connectionModeResolverPrefersConfigModeOverDefaults() { + let defaults = self.makeDefaults() + defaults.set("remote", forKey: connectionModeKey) + + let root: [String: Any] = [ + "gateway": [ + "mode": " local ", + ], + ] + + let resolved = ConnectionModeResolver.resolve(root: root, defaults: defaults) + #expect(resolved.mode == .local) + } + + @Test func connectionModeResolverTrimsConfigMode() { + let defaults = self.makeDefaults() + defaults.set("local", forKey: connectionModeKey) + + let root: [String: Any] = [ + "gateway": [ + "mode": " remote ", + ], + ] + + let resolved = ConnectionModeResolver.resolve(root: root, defaults: defaults) + #expect(resolved.mode == .remote) + } + + @Test func connectionModeResolverFallsBackToDefaultsWhenMissingConfig() { + let defaults = self.makeDefaults() + defaults.set("remote", forKey: connectionModeKey) + + let resolved = ConnectionModeResolver.resolve(root: [:], defaults: defaults) + #expect(resolved.mode == .remote) + } + + @Test func connectionModeResolverFallsBackToDefaultsOnUnknownConfig() { + let defaults = self.makeDefaults() + defaults.set("local", forKey: connectionModeKey) + + let root: [String: Any] = [ + "gateway": [ + "mode": "staging", + ], + ] + + let resolved = ConnectionModeResolver.resolve(root: root, defaults: defaults) + #expect(resolved.mode == .local) + } + + @Test func connectionModeResolverPrefersRemoteURLWhenModeMissing() { + let defaults = self.makeDefaults() + defaults.set("local", forKey: connectionModeKey) + + let root: [String: Any] = [ + "gateway": [ + "remote": [ + "url": " ws://umbrel:18789 ", + ], + ], + ] + + let resolved = ConnectionModeResolver.resolve(root: root, defaults: defaults) + #expect(resolved.mode == .remote) + } + + @Test func resolveLocalGatewayHostUsesLoopbackForAutoEvenWithTailnet() { + let host = GatewayEndpointStore._testResolveLocalGatewayHost( + bindMode: "auto", + tailscaleIP: "100.64.1.2") + #expect(host == "127.0.0.1") + } + + @Test func resolveLocalGatewayHostUsesLoopbackForAutoWithoutTailnet() { + let host = GatewayEndpointStore._testResolveLocalGatewayHost( + bindMode: "auto", + tailscaleIP: nil) + #expect(host == "127.0.0.1") + } + + @Test func resolveLocalGatewayHostPrefersTailnetForTailnetMode() { + let host = GatewayEndpointStore._testResolveLocalGatewayHost( + bindMode: "tailnet", + tailscaleIP: "100.64.1.5") + #expect(host == "100.64.1.5") + } + + @Test func resolveLocalGatewayHostFallsBackToLoopbackForTailnetMode() { + let host = GatewayEndpointStore._testResolveLocalGatewayHost( + bindMode: "tailnet", + tailscaleIP: nil) + #expect(host == "127.0.0.1") + } + + @Test func resolveLocalGatewayHostUsesCustomBindHost() { + let host = GatewayEndpointStore._testResolveLocalGatewayHost( + bindMode: "custom", + tailscaleIP: "100.64.1.9", + customBindHost: "192.168.1.10") + #expect(host == "192.168.1.10") + } + + @Test func normalizeGatewayUrlAddsDefaultPortForWs() { + let url = GatewayRemoteConfig.normalizeGatewayUrl("ws://gateway") + #expect(url?.port == 18789) + #expect(url?.absoluteString == "ws://gateway:18789") + } +} diff --git a/apps/macos/Tests/OpenClawIPCTests/GatewayEnvironmentTests.swift b/apps/macos/Tests/OpenClawIPCTests/GatewayEnvironmentTests.swift new file mode 100644 index 0000000000000000000000000000000000000000..32dcbb737f9de27d2150541f9648c805f98e7774 --- /dev/null +++ b/apps/macos/Tests/OpenClawIPCTests/GatewayEnvironmentTests.swift @@ -0,0 +1,57 @@ +import Foundation +import Testing +@testable import OpenClaw + +@Suite struct GatewayEnvironmentTests { + @Test func semverParsesCommonForms() { + #expect(Semver.parse("1.2.3") == Semver(major: 1, minor: 2, patch: 3)) + #expect(Semver.parse(" v1.2.3 \n") == Semver(major: 1, minor: 2, patch: 3)) + #expect(Semver.parse("v2.0.0") == Semver(major: 2, minor: 0, patch: 0)) + #expect(Semver.parse("3.4.5-beta.1") == Semver(major: 3, minor: 4, patch: 5)) // prerelease suffix stripped + #expect(Semver.parse("2026.1.11-4") == Semver(major: 2026, minor: 1, patch: 11)) // build suffix stripped + #expect(Semver.parse("1.0.5+build.123") == Semver(major: 1, minor: 0, patch: 5)) // metadata suffix stripped + #expect(Semver.parse("v1.2.3+build.9") == Semver(major: 1, minor: 2, patch: 3)) + #expect(Semver.parse("1.2.3+build.123") == Semver(major: 1, minor: 2, patch: 3)) + #expect(Semver.parse("1.2.3-rc.1+build.7") == Semver(major: 1, minor: 2, patch: 3)) + #expect(Semver.parse("v1.2.3-rc.1") == Semver(major: 1, minor: 2, patch: 3)) + #expect(Semver.parse("1.2.0") == Semver(major: 1, minor: 2, patch: 0)) + #expect(Semver.parse(nil) == nil) + #expect(Semver.parse("invalid") == nil) + #expect(Semver.parse("1.2") == nil) + #expect(Semver.parse("1.2.x") == nil) + } + + @Test func semverCompatibilityRequiresSameMajorAndNotOlder() { + let required = Semver(major: 2, minor: 1, patch: 0) + #expect(Semver(major: 2, minor: 1, patch: 0).compatible(with: required)) + #expect(Semver(major: 2, minor: 2, patch: 0).compatible(with: required)) + #expect(Semver(major: 2, minor: 1, patch: 1).compatible(with: required)) + #expect(Semver(major: 2, minor: 0, patch: 9).compatible(with: required) == false) + #expect(Semver(major: 3, minor: 0, patch: 0).compatible(with: required) == false) + #expect(Semver(major: 1, minor: 9, patch: 9).compatible(with: required) == false) + } + + @Test func gatewayPortDefaultsAndRespectsOverride() async { + let configPath = TestIsolation.tempConfigPath() + await TestIsolation.withIsolatedState( + env: ["OPENCLAW_CONFIG_PATH": configPath], + defaults: ["gatewayPort": nil]) + { + let defaultPort = GatewayEnvironment.gatewayPort() + #expect(defaultPort == 18789) + + UserDefaults.standard.set(19999, forKey: "gatewayPort") + defer { UserDefaults.standard.removeObject(forKey: "gatewayPort") } + #expect(GatewayEnvironment.gatewayPort() == 19999) + } + } + + @Test func expectedGatewayVersionFromStringUsesParser() { + #expect(GatewayEnvironment.expectedGatewayVersion(from: "v9.1.2") == Semver(major: 9, minor: 1, patch: 2)) + #expect(GatewayEnvironment.expectedGatewayVersion(from: "2026.1.11-4") == Semver( + major: 2026, + minor: 1, + patch: 11)) + #expect(GatewayEnvironment.expectedGatewayVersion(from: nil) == nil) + } +} diff --git a/apps/macos/Tests/OpenClawIPCTests/GatewayFrameDecodeTests.swift b/apps/macos/Tests/OpenClawIPCTests/GatewayFrameDecodeTests.swift new file mode 100644 index 0000000000000000000000000000000000000000..bda8ff0e44327b9dc593830d465a2819740e446b --- /dev/null +++ b/apps/macos/Tests/OpenClawIPCTests/GatewayFrameDecodeTests.swift @@ -0,0 +1,98 @@ +import OpenClawProtocol +import Foundation +import Testing + +@Suite struct GatewayFrameDecodeTests { + @Test func decodesEventFrameWithAnyCodablePayload() throws { + let json = """ + { + "type": "event", + "event": "presence", + "payload": { "foo": "bar", "count": 1 }, + "seq": 7 + } + """ + + let frame = try JSONDecoder().decode(GatewayFrame.self, from: Data(json.utf8)) + + #expect({ + if case .event = frame { true } else { false } + }(), "expected .event frame") + + guard case let .event(evt) = frame else { + return + } + + let payload = evt.payload?.value as? [String: AnyCodable] + #expect(payload?["foo"]?.value as? String == "bar") + #expect(payload?["count"]?.value as? Int == 1) + #expect(evt.seq == 7) + } + + @Test func decodesRequestFrameWithNestedParams() throws { + let json = """ + { + "type": "req", + "id": "1", + "method": "agent.send", + "params": { + "text": "hi", + "items": [1, null, {"ok": true}], + "meta": { "count": 2 } + } + } + """ + + let frame = try JSONDecoder().decode(GatewayFrame.self, from: Data(json.utf8)) + + #expect({ + if case .req = frame { true } else { false } + }(), "expected .req frame") + + guard case let .req(req) = frame else { + return + } + + let params = req.params?.value as? [String: AnyCodable] + #expect(params?["text"]?.value as? String == "hi") + + let items = params?["items"]?.value as? [AnyCodable] + #expect(items?.count == 3) + #expect(items?[0].value as? Int == 1) + #expect(items?[1].value is NSNull) + + let item2 = items?[2].value as? [String: AnyCodable] + #expect(item2?["ok"]?.value as? Bool == true) + + let meta = params?["meta"]?.value as? [String: AnyCodable] + #expect(meta?["count"]?.value as? Int == 2) + } + + @Test func decodesUnknownFrameAndPreservesRaw() throws { + let json = """ + { + "type": "made-up", + "foo": "bar", + "count": 1, + "nested": { "ok": true } + } + """ + + let frame = try JSONDecoder().decode(GatewayFrame.self, from: Data(json.utf8)) + + #expect({ + if case .unknown = frame { true } else { false } + }(), "expected .unknown frame") + + guard case let .unknown(type, raw) = frame else { + return + } + + #expect(type == "made-up") + #expect(raw["type"]?.value as? String == "made-up") + #expect(raw["foo"]?.value as? String == "bar") + #expect(raw["count"]?.value as? Int == 1) + let nested = raw["nested"]?.value as? [String: AnyCodable] + #expect(nested?["ok"]?.value as? Bool == true) + } +} diff --git a/apps/macos/Tests/OpenClawIPCTests/GatewayLaunchAgentManagerTests.swift b/apps/macos/Tests/OpenClawIPCTests/GatewayLaunchAgentManagerTests.swift new file mode 100644 index 0000000000000000000000000000000000000000..685db8185fcf3b44d9805b011a1f38cb159d6d57 --- /dev/null +++ b/apps/macos/Tests/OpenClawIPCTests/GatewayLaunchAgentManagerTests.swift @@ -0,0 +1,41 @@ +import Foundation +import Testing +@testable import OpenClaw + +@Suite struct GatewayLaunchAgentManagerTests { + @Test func launchAgentPlistSnapshotParsesArgsAndEnv() throws { + let url = FileManager().temporaryDirectory + .appendingPathComponent("openclaw-launchd-\(UUID().uuidString).plist") + let plist: [String: Any] = [ + "ProgramArguments": ["openclaw", "gateway-daemon", "--port", "18789", "--bind", "loopback"], + "EnvironmentVariables": [ + "OPENCLAW_GATEWAY_TOKEN": " secret ", + "OPENCLAW_GATEWAY_PASSWORD": "pw", + ], + ] + let data = try PropertyListSerialization.data(fromPropertyList: plist, format: .xml, options: 0) + try data.write(to: url, options: [.atomic]) + defer { try? FileManager().removeItem(at: url) } + + let snapshot = try #require(LaunchAgentPlist.snapshot(url: url)) + #expect(snapshot.port == 18789) + #expect(snapshot.bind == "loopback") + #expect(snapshot.token == "secret") + #expect(snapshot.password == "pw") + } + + @Test func launchAgentPlistSnapshotAllowsMissingBind() throws { + let url = FileManager().temporaryDirectory + .appendingPathComponent("openclaw-launchd-\(UUID().uuidString).plist") + let plist: [String: Any] = [ + "ProgramArguments": ["openclaw", "gateway-daemon", "--port", "18789"], + ] + let data = try PropertyListSerialization.data(fromPropertyList: plist, format: .xml, options: 0) + try data.write(to: url, options: [.atomic]) + defer { try? FileManager().removeItem(at: url) } + + let snapshot = try #require(LaunchAgentPlist.snapshot(url: url)) + #expect(snapshot.port == 18789) + #expect(snapshot.bind == nil) + } +} diff --git a/apps/macos/Tests/OpenClawIPCTests/GatewayProcessManagerTests.swift b/apps/macos/Tests/OpenClawIPCTests/GatewayProcessManagerTests.swift new file mode 100644 index 0000000000000000000000000000000000000000..f8b226ab277fd1057238c03507974ac2393fd503 --- /dev/null +++ b/apps/macos/Tests/OpenClawIPCTests/GatewayProcessManagerTests.swift @@ -0,0 +1,147 @@ +import OpenClawKit +import Foundation +import os +import Testing +@testable import OpenClaw + +@Suite(.serialized) +@MainActor +struct GatewayProcessManagerTests { + private final class FakeWebSocketTask: WebSocketTasking, @unchecked Sendable { + private let connectRequestID = OSAllocatedUnfairLock(initialState: nil) + private let pendingReceiveHandler = + OSAllocatedUnfairLock<(@Sendable (Result) + -> Void)?>(initialState: nil) + private let cancelCount = OSAllocatedUnfairLock(initialState: 0) + private let sendCount = OSAllocatedUnfairLock(initialState: 0) + + var state: URLSessionTask.State = .suspended + + func resume() { + self.state = .running + } + + func cancel(with closeCode: URLSessionWebSocketTask.CloseCode, reason: Data?) { + _ = (closeCode, reason) + self.state = .canceling + self.cancelCount.withLock { $0 += 1 } + let handler = self.pendingReceiveHandler.withLock { handler in + defer { handler = nil } + return handler + } + handler?(Result.failure(URLError(.cancelled))) + } + + func send(_ message: URLSessionWebSocketTask.Message) async throws { + let currentSendCount = self.sendCount.withLock { count in + defer { count += 1 } + return count + } + + if currentSendCount == 0 { + guard case let .data(data) = message else { return } + if let obj = try? JSONSerialization.jsonObject(with: data) as? [String: Any], + (obj["type"] as? String) == "req", + (obj["method"] as? String) == "connect", + let id = obj["id"] as? String + { + self.connectRequestID.withLock { $0 = id } + } + return + } + + guard case let .data(data) = message else { return } + guard + let obj = try? JSONSerialization.jsonObject(with: data) as? [String: Any], + (obj["type"] as? String) == "req", + let id = obj["id"] as? String + else { + return + } + + let response = Self.responseData(id: id) + let handler = self.pendingReceiveHandler.withLock { $0 } + handler?(Result.success(.data(response))) + } + + func receive() async throws -> URLSessionWebSocketTask.Message { + let id = self.connectRequestID.withLock { $0 } ?? "connect" + return .data(Self.connectOkData(id: id)) + } + + func receive( + completionHandler: @escaping @Sendable (Result) -> Void) + { + self.pendingReceiveHandler.withLock { $0 = completionHandler } + } + + private static func connectOkData(id: String) -> Data { + let json = """ + { + "type": "res", + "id": "\(id)", + "ok": true, + "payload": { + "type": "hello-ok", + "protocol": 2, + "server": { "version": "test", "connId": "test" }, + "features": { "methods": [], "events": [] }, + "snapshot": { + "presence": [ { "ts": 1 } ], + "health": {}, + "stateVersion": { "presence": 0, "health": 0 }, + "uptimeMs": 0 + }, + "policy": { "maxPayload": 1, "maxBufferedBytes": 1, "tickIntervalMs": 30000 } + } + } + """ + return Data(json.utf8) + } + + private static func responseData(id: String) -> Data { + let json = """ + { + "type": "res", + "id": "\(id)", + "ok": true, + "payload": { "ok": true } + } + """ + return Data(json.utf8) + } + } + + private final class FakeWebSocketSession: WebSocketSessioning, @unchecked Sendable { + private let tasks = OSAllocatedUnfairLock(initialState: [FakeWebSocketTask]()) + + func makeWebSocketTask(url: URL) -> WebSocketTaskBox { + _ = url + let task = FakeWebSocketTask() + self.tasks.withLock { $0.append(task) } + return WebSocketTaskBox(task: task) + } + } + + @Test func clearsLastFailureWhenHealthSucceeds() async { + let session = FakeWebSocketSession() + let url = URL(string: "ws://example.invalid")! + let connection = GatewayConnection( + configProvider: { (url: url, token: nil, password: nil) }, + sessionBox: WebSocketSessionBox(session: session)) + + let manager = GatewayProcessManager.shared + manager.setTestingConnection(connection) + manager.setTestingDesiredActive(true) + manager.setTestingLastFailureReason("health failed") + defer { + manager.setTestingConnection(nil) + manager.setTestingDesiredActive(false) + manager.setTestingLastFailureReason(nil) + } + + let ready = await manager.waitForGatewayReady(timeout: 0.5) + #expect(ready) + #expect(manager.lastFailureReason == nil) + } +} diff --git a/apps/macos/Tests/OpenClawIPCTests/HealthDecodeTests.swift b/apps/macos/Tests/OpenClawIPCTests/HealthDecodeTests.swift new file mode 100644 index 0000000000000000000000000000000000000000..f6b65b154d13863912632ee9741c8a9f9f642f29 --- /dev/null +++ b/apps/macos/Tests/OpenClawIPCTests/HealthDecodeTests.swift @@ -0,0 +1,32 @@ +import Foundation +import Testing +@testable import OpenClaw + +@Suite struct HealthDecodeTests { + private let sampleJSON: String = // minimal but complete payload + """ + {"ts":1733622000,"durationMs":420,"channels":{"whatsapp":{"linked":true,"authAgeMs":120000},"telegram":{"configured":true,"probe":{"ok":true,"elapsedMs":800}}},"channelOrder":["whatsapp","telegram"],"heartbeatSeconds":60,"sessions":{"path":"/tmp/sessions.json","count":1,"recent":[{"key":"abc","updatedAt":1733621900,"age":120000}]}} + """ + + @Test func decodesCleanJSON() async throws { + let data = Data(sampleJSON.utf8) + let snap = decodeHealthSnapshot(from: data) + + #expect(snap?.channels["whatsapp"]?.linked == true) + #expect(snap?.sessions.count == 1) + } + + @Test func decodesWithLeadingNoise() async throws { + let noisy = "debug: something logged\n" + self.sampleJSON + "\ntrailer" + let snap = decodeHealthSnapshot(from: Data(noisy.utf8)) + + #expect(snap?.channels["telegram"]?.probe?.elapsedMs == 800) + } + + @Test func failsWithoutBraces() async throws { + let data = Data("no json here".utf8) + let snap = decodeHealthSnapshot(from: data) + + #expect(snap == nil) + } +} diff --git a/apps/macos/Tests/OpenClawIPCTests/HealthStoreStateTests.swift b/apps/macos/Tests/OpenClawIPCTests/HealthStoreStateTests.swift new file mode 100644 index 0000000000000000000000000000000000000000..ca2601cf6fb249db37eed117aa5b9f54f17995f5 --- /dev/null +++ b/apps/macos/Tests/OpenClawIPCTests/HealthStoreStateTests.swift @@ -0,0 +1,42 @@ +import Foundation +import Testing +@testable import OpenClaw + +@Suite struct HealthStoreStateTests { + @Test @MainActor func linkedChannelProbeFailureDegradesState() async throws { + let snap = HealthSnapshot( + ok: true, + ts: 0, + durationMs: 1, + channels: [ + "whatsapp": .init( + configured: true, + linked: true, + authAgeMs: 1, + probe: .init( + ok: false, + status: 503, + error: "gateway connect failed", + elapsedMs: 12, + bot: nil, + webhook: nil), + lastProbeAt: 0), + ], + channelOrder: ["whatsapp"], + channelLabels: ["whatsapp": "WhatsApp"], + heartbeatSeconds: 60, + sessions: .init(path: "/tmp/sessions.json", count: 0, recent: [])) + + let store = HealthStore.shared + store.__setSnapshotForTest(snap, lastError: nil) + + switch store.state { + case let .degraded(message): + #expect(!message.isEmpty) + default: + Issue.record("Expected degraded state when probe fails for linked channel") + } + + #expect(store.summaryLine.contains("probe degraded")) + } +} diff --git a/apps/macos/Tests/OpenClawIPCTests/HoverHUDControllerTests.swift b/apps/macos/Tests/OpenClawIPCTests/HoverHUDControllerTests.swift new file mode 100644 index 0000000000000000000000000000000000000000..eff3ee6d814e1cec89a690991515336503ee78ed --- /dev/null +++ b/apps/macos/Tests/OpenClawIPCTests/HoverHUDControllerTests.swift @@ -0,0 +1,26 @@ +import AppKit +import Testing +@testable import OpenClaw + +@Suite(.serialized) +@MainActor +struct HoverHUDControllerTests { + @Test func hoverHUDControllerPresentsAndDismisses() async { + let controller = HoverHUDController() + controller.setSuppressed(false) + + controller.statusItemHoverChanged( + inside: true, + anchorProvider: { NSRect(x: 10, y: 10, width: 24, height: 24) }) + try? await Task.sleep(nanoseconds: 260_000_000) + + controller.panelHoverChanged(inside: true) + controller.panelHoverChanged(inside: false) + controller.statusItemHoverChanged( + inside: false, + anchorProvider: { NSRect(x: 10, y: 10, width: 24, height: 24) }) + + controller.dismiss(reason: "test") + controller.setSuppressed(true) + } +} diff --git a/apps/macos/Tests/OpenClawIPCTests/InstancesSettingsSmokeTests.swift b/apps/macos/Tests/OpenClawIPCTests/InstancesSettingsSmokeTests.swift new file mode 100644 index 0000000000000000000000000000000000000000..c43982ee82bfdd9e0a23a29e977967155f7a8faa --- /dev/null +++ b/apps/macos/Tests/OpenClawIPCTests/InstancesSettingsSmokeTests.swift @@ -0,0 +1,59 @@ +import Testing +@testable import OpenClaw + +@Suite(.serialized) +@MainActor +struct InstancesSettingsSmokeTests { + @Test func instancesSettingsBuildsBodyWithMultipleInstances() { + let store = InstancesStore(isPreview: true) + store.statusMessage = "Loaded" + store.instances = [ + InstanceInfo( + id: "macbook", + host: "macbook-pro", + ip: "10.0.0.2", + version: "1.2.3", + platform: "macOS 15.1", + deviceFamily: "Mac", + modelIdentifier: "MacBookPro18,1", + lastInputSeconds: 15, + mode: "local", + reason: "heartbeat", + text: "MacBook Pro local", + ts: 1_700_000_000_000), + InstanceInfo( + id: "android", + host: "pixel", + ip: "10.0.0.3", + version: "2.0.0", + platform: "Android 14", + deviceFamily: "Android", + modelIdentifier: nil, + lastInputSeconds: 120, + mode: "node", + reason: "presence", + text: "Android node", + ts: 1_700_000_100_000), + InstanceInfo( + id: "gateway", + host: "gateway", + ip: "10.0.0.4", + version: "3.0.0", + platform: "iOS 18", + deviceFamily: nil, + modelIdentifier: nil, + lastInputSeconds: nil, + mode: "gateway", + reason: "gateway", + text: "Gateway", + ts: 1_700_000_200_000), + ] + + let view = InstancesSettings(store: store) + _ = view.body + } + + @Test func instancesSettingsExercisesHelpers() { + InstancesSettings.exerciseForTesting() + } +} diff --git a/apps/macos/Tests/OpenClawIPCTests/InstancesStoreTests.swift b/apps/macos/Tests/OpenClawIPCTests/InstancesStoreTests.swift new file mode 100644 index 0000000000000000000000000000000000000000..f148c35fb21e0ebb2aa7c42aacb100dacf786d7f --- /dev/null +++ b/apps/macos/Tests/OpenClawIPCTests/InstancesStoreTests.swift @@ -0,0 +1,36 @@ +import OpenClawProtocol +import Testing +@testable import OpenClaw + +@Suite struct InstancesStoreTests { + @Test + @MainActor + func presenceEventPayloadDecodesViaJSONEncoder() { + // Build a payload that mirrors the gateway's presence event shape: + // { "presence": [ PresenceEntry ] } + let entry: [String: OpenClawProtocol.AnyCodable] = [ + "host": .init("gw"), + "ip": .init("10.0.0.1"), + "version": .init("2.0.0"), + "mode": .init("gateway"), + "lastInputSeconds": .init(5), + "reason": .init("test"), + "text": .init("Gateway node"), + "ts": .init(1_730_000_000), + ] + let payloadMap: [String: OpenClawProtocol.AnyCodable] = [ + "presence": .init([OpenClawProtocol.AnyCodable(entry)]), + ] + let payload = OpenClawProtocol.AnyCodable(payloadMap) + + let store = InstancesStore(isPreview: true) + store.handlePresenceEventPayload(payload) + + #expect(store.instances.count == 1) + let instance = store.instances.first + #expect(instance?.host == "gw") + #expect(instance?.ip == "10.0.0.1") + #expect(instance?.mode == "gateway") + #expect(instance?.reason == "test") + } +} diff --git a/apps/macos/Tests/OpenClawIPCTests/LogLocatorTests.swift b/apps/macos/Tests/OpenClawIPCTests/LogLocatorTests.swift new file mode 100644 index 0000000000000000000000000000000000000000..6f7fc5dc0165cfb45a377fd8c049d305c6ab4e8c --- /dev/null +++ b/apps/macos/Tests/OpenClawIPCTests/LogLocatorTests.swift @@ -0,0 +1,24 @@ +import Darwin +import Foundation +import Testing +@testable import OpenClaw + +@Suite struct LogLocatorTests { + @Test func launchdGatewayLogPathEnsuresTmpDirExists() throws { + let fm = FileManager() + let baseDir = URL(fileURLWithPath: NSTemporaryDirectory(), isDirectory: true) + let logDir = baseDir.appendingPathComponent("openclaw-tests-\(UUID().uuidString)") + + setenv("OPENCLAW_LOG_DIR", logDir.path, 1) + defer { + unsetenv("OPENCLAW_LOG_DIR") + try? fm.removeItem(at: logDir) + } + + _ = LogLocator.launchdGatewayLogPath + + var isDir: ObjCBool = false + #expect(fm.fileExists(atPath: logDir.path, isDirectory: &isDir)) + #expect(isDir.boolValue == true) + } +} diff --git a/apps/macos/Tests/OpenClawIPCTests/LowCoverageHelperTests.swift b/apps/macos/Tests/OpenClawIPCTests/LowCoverageHelperTests.swift new file mode 100644 index 0000000000000000000000000000000000000000..174dc1d134c770ac0b8203d3da5c4e9dbc1c54a5 --- /dev/null +++ b/apps/macos/Tests/OpenClawIPCTests/LowCoverageHelperTests.swift @@ -0,0 +1,215 @@ +import AppKit +import OpenClawProtocol +import Foundation +import Testing + +@testable import OpenClaw + +@Suite(.serialized) +struct LowCoverageHelperTests { + private typealias ProtoAnyCodable = OpenClawProtocol.AnyCodable + + @Test func anyCodableHelperAccessors() throws { + let payload: [String: ProtoAnyCodable] = [ + "title": ProtoAnyCodable("Hello"), + "flag": ProtoAnyCodable(true), + "count": ProtoAnyCodable(3), + "ratio": ProtoAnyCodable(1.25), + "list": ProtoAnyCodable([ProtoAnyCodable("a"), ProtoAnyCodable(2)]), + ] + let any = ProtoAnyCodable(payload) + let dict = try #require(any.dictionaryValue) + #expect(dict["title"]?.stringValue == "Hello") + #expect(dict["flag"]?.boolValue == true) + #expect(dict["count"]?.intValue == 3) + #expect(dict["ratio"]?.doubleValue == 1.25) + #expect(dict["list"]?.arrayValue?.count == 2) + + let foundation = any.foundationValue as? [String: Any] + #expect((foundation?["title"] as? String) == "Hello") + } + + @Test func attributedStringStripsForegroundColor() { + let text = NSMutableAttributedString(string: "Test") + text.addAttribute(.foregroundColor, value: NSColor.red, range: NSRange(location: 0, length: 4)) + let stripped = text.strippingForegroundColor() + let color = stripped.attribute(.foregroundColor, at: 0, effectiveRange: nil) + #expect(color == nil) + } + + @Test func viewMetricsReduceWidth() { + let value = ViewMetricsTesting.reduceWidth(current: 120, next: 180) + #expect(value == 180) + } + + @Test func shellExecutorHandlesEmptyCommand() async { + let result = await ShellExecutor.runDetailed(command: [], cwd: nil, env: nil, timeout: nil) + #expect(result.success == false) + #expect(result.errorMessage != nil) + } + + @Test func shellExecutorRunsCommand() async { + let result = await ShellExecutor.runDetailed(command: ["/bin/echo", "ok"], cwd: nil, env: nil, timeout: 2) + #expect(result.success == true) + #expect(result.stdout.contains("ok") || result.stderr.contains("ok")) + } + + @Test func shellExecutorTimesOut() async { + let result = await ShellExecutor.runDetailed(command: ["/bin/sleep", "1"], cwd: nil, env: nil, timeout: 0.05) + #expect(result.timedOut == true) + } + + @Test func shellExecutorDrainsStdoutAndStderr() async { + let script = """ + i=0 + while [ $i -lt 2000 ]; do + echo "stdout-$i" + echo "stderr-$i" 1>&2 + i=$((i+1)) + done + """ + let result = await ShellExecutor.runDetailed( + command: ["/bin/sh", "-c", script], + cwd: nil, + env: nil, + timeout: 2) + #expect(result.success == true) + #expect(result.stdout.contains("stdout-1999")) + #expect(result.stderr.contains("stderr-1999")) + } + + @Test func nodeInfoCodableRoundTrip() throws { + let info = NodeInfo( + nodeId: "node-1", + displayName: "Node One", + platform: "macOS", + version: "1.0", + coreVersion: "1.0-core", + uiVersion: "1.0-ui", + deviceFamily: "Mac", + modelIdentifier: "MacBookPro", + remoteIp: "192.168.1.2", + caps: ["chat"], + commands: ["send"], + permissions: ["send": true], + paired: true, + connected: false) + let data = try JSONEncoder().encode(info) + let decoded = try JSONDecoder().decode(NodeInfo.self, from: data) + #expect(decoded.nodeId == "node-1") + #expect(decoded.isPaired == true) + #expect(decoded.isConnected == false) + } + + @Test @MainActor func presenceReporterHelpers() { + let summary = PresenceReporter._testComposePresenceSummary(mode: "local", reason: "test") + #expect(summary.contains("mode local")) + #expect(!PresenceReporter._testAppVersionString().isEmpty) + #expect(!PresenceReporter._testPlatformString().isEmpty) + _ = PresenceReporter._testLastInputSeconds() + _ = PresenceReporter._testPrimaryIPv4Address() + } + + @Test func portGuardianParsesListenersAndBuildsReports() { + let output = """ + p123 + cnode + uuser + p456 + cssh + uroot + """ + let listeners = PortGuardian._testParseListeners(output) + #expect(listeners.count == 2) + #expect(listeners[0].command == "node") + #expect(listeners[1].command == "ssh") + + let okReport = PortGuardian._testBuildReport( + port: 18789, + mode: .local, + listeners: [(pid: 1, command: "node", fullCommand: "node", user: "me")]) + #expect(okReport.offenders.isEmpty) + + let badReport = PortGuardian._testBuildReport( + port: 18789, + mode: .local, + listeners: [(pid: 2, command: "python", fullCommand: "python", user: "me")]) + #expect(!badReport.offenders.isEmpty) + + let emptyReport = PortGuardian._testBuildReport(port: 18789, mode: .local, listeners: []) + #expect(emptyReport.summary.contains("Nothing is listening")) + } + + @Test @MainActor func canvasSchemeHandlerResolvesFilesAndErrors() throws { + let root = FileManager().temporaryDirectory + .appendingPathComponent("canvas-\(UUID().uuidString)", isDirectory: true) + defer { try? FileManager().removeItem(at: root) } + try FileManager().createDirectory(at: root, withIntermediateDirectories: true) + let session = root.appendingPathComponent("main", isDirectory: true) + try FileManager().createDirectory(at: session, withIntermediateDirectories: true) + + let index = session.appendingPathComponent("index.html") + try "

Hello

".write(to: index, atomically: true, encoding: .utf8) + + let handler = CanvasSchemeHandler(root: root) + let url = try #require(CanvasScheme.makeURL(session: "main", path: "index.html")) + let response = handler._testResponse(for: url) + #expect(response.mime == "text/html") + #expect(String(data: response.data, encoding: .utf8)?.contains("Hello") == true) + + let invalid = URL(string: "https://example.com")! + let invalidResponse = handler._testResponse(for: invalid) + #expect(invalidResponse.mime == "text/html") + + let missing = try #require(CanvasScheme.makeURL(session: "missing", path: "/")) + let missingResponse = handler._testResponse(for: missing) + #expect(missingResponse.mime == "text/html") + + #expect(handler._testTextEncodingName(for: "text/html") == "utf-8") + #expect(handler._testTextEncodingName(for: "application/octet-stream") == nil) + } + + @Test @MainActor func menuContextCardInjectorInsertsAndFindsIndex() { + let injector = MenuContextCardInjector() + let menu = NSMenu() + menu.minimumWidth = 280 + menu.addItem(NSMenuItem(title: "Active", action: nil, keyEquivalent: "")) + menu.addItem(.separator()) + menu.addItem(NSMenuItem(title: "Send Heartbeats", action: nil, keyEquivalent: "")) + menu.addItem(NSMenuItem(title: "Quit", action: nil, keyEquivalent: "q")) + + let idx = injector._testFindInsertIndex(in: menu) + #expect(idx == 1) + #expect(injector._testInitialCardWidth(for: menu) >= 300) + + injector._testSetCache(rows: [SessionRow.previewRows[0]], errorText: nil, updatedAt: Date()) + injector.menuWillOpen(menu) + injector.menuDidClose(menu) + + let fallbackMenu = NSMenu() + fallbackMenu.addItem(NSMenuItem(title: "First", action: nil, keyEquivalent: "")) + #expect(injector._testFindInsertIndex(in: fallbackMenu) == 1) + } + + @Test @MainActor func canvasWindowHelperFunctions() { + #expect(CanvasWindowController._testSanitizeSessionKey(" main ") == "main") + #expect(CanvasWindowController._testSanitizeSessionKey("bad/..") == "bad___") + #expect(CanvasWindowController._testJSOptionalStringLiteral(nil) == "null") + + let rect = NSRect(x: 10, y: 12, width: 400, height: 420) + let key = CanvasWindowController._testStoredFrameKey(sessionKey: "test") + let loaded = CanvasWindowController._testStoreAndLoadFrame(sessionKey: "test", frame: rect) + UserDefaults.standard.removeObject(forKey: key) + #expect(loaded?.size.width == rect.size.width) + + let parsed = CanvasWindowController._testParseIPv4("192.168.1.2") + #expect(parsed != nil) + if let parsed { + #expect(CanvasWindowController._testIsLocalNetworkIPv4(parsed)) + } + + let url = URL(string: "http://192.168.1.2")! + #expect(CanvasWindowController._testIsLocalNetworkCanvasURL(url)) + #expect(CanvasWindowController._testParseIPv4("not-an-ip") == nil) + } +} diff --git a/apps/macos/Tests/OpenClawIPCTests/LowCoverageViewSmokeTests.swift b/apps/macos/Tests/OpenClawIPCTests/LowCoverageViewSmokeTests.swift new file mode 100644 index 0000000000000000000000000000000000000000..aea7f61679b82155dd9b0add88d6b3763c025571 --- /dev/null +++ b/apps/macos/Tests/OpenClawIPCTests/LowCoverageViewSmokeTests.swift @@ -0,0 +1,99 @@ +import AppKit +import OpenClawProtocol +import SwiftUI +import Testing + +@testable import OpenClaw + +@Suite(.serialized) +@MainActor +struct LowCoverageViewSmokeTests { + @Test func contextMenuCardBuildsBody() { + let loading = ContextMenuCardView(rows: [], statusText: "Loading…", isLoading: true) + _ = loading.body + + let empty = ContextMenuCardView(rows: [], statusText: nil, isLoading: false) + _ = empty.body + + let withRows = ContextMenuCardView(rows: SessionRow.previewRows, statusText: nil, isLoading: false) + _ = withRows.body + } + + @Test func settingsToggleRowBuildsBody() { + var flag = false + let binding = Binding(get: { flag }, set: { flag = $0 }) + let view = SettingsToggleRow(title: "Enable", subtitle: "Detail", binding: binding) + _ = view.body + } + + @Test func voiceWakeTestCardBuildsBodyAcrossStates() { + var state = VoiceWakeTestState.idle + var isTesting = false + let stateBinding = Binding(get: { state }, set: { state = $0 }) + let testingBinding = Binding(get: { isTesting }, set: { isTesting = $0 }) + + _ = VoiceWakeTestCard(testState: stateBinding, isTesting: testingBinding, onToggle: {}).body + + state = .hearing("hello") + _ = VoiceWakeTestCard(testState: stateBinding, isTesting: testingBinding, onToggle: {}).body + + state = .detected("command") + isTesting = true + _ = VoiceWakeTestCard(testState: stateBinding, isTesting: testingBinding, onToggle: {}).body + + state = .failed("No mic") + _ = VoiceWakeTestCard(testState: stateBinding, isTesting: testingBinding, onToggle: {}).body + } + + @Test func agentEventsWindowBuildsBodyWithEvent() { + AgentEventStore.shared.clear() + let sample = ControlAgentEvent( + runId: "run-1", + seq: 1, + stream: "tool", + ts: Date().timeIntervalSince1970 * 1000, + data: ["phase": AnyCodable("start"), "name": AnyCodable("test")], + summary: nil) + AgentEventStore.shared.append(sample) + _ = AgentEventsWindow().body + AgentEventStore.shared.clear() + } + + @Test func notifyOverlayPresentsAndDismisses() async { + let controller = NotifyOverlayController() + controller.present(title: "Hello", body: "World", autoDismissAfter: 0) + controller.present(title: "Updated", body: "Again", autoDismissAfter: 0) + controller.dismiss() + try? await Task.sleep(nanoseconds: 250_000_000) + } + + @Test func visualEffectViewHostsInNSHostingView() { + let hosting = NSHostingView(rootView: VisualEffectView(material: .sidebar)) + _ = hosting.fittingSize + hosting.rootView = VisualEffectView(material: .popover, emphasized: true) + _ = hosting.fittingSize + } + + @Test func menuHostedItemHostsContent() { + let view = MenuHostedItem(width: 240, rootView: AnyView(Text("Menu"))) + let hosting = NSHostingView(rootView: view) + _ = hosting.fittingSize + hosting.rootView = MenuHostedItem(width: 320, rootView: AnyView(Text("Updated"))) + _ = hosting.fittingSize + } + + @Test func dockIconManagerUpdatesVisibility() { + _ = NSApplication.shared + UserDefaults.standard.set(false, forKey: showDockIconKey) + DockIconManager.shared.updateDockVisibility() + DockIconManager.shared.temporarilyShowDock() + } + + @Test func voiceWakeSettingsExercisesHelpers() { + VoiceWakeSettings.exerciseForTesting() + } + + @Test func debugSettingsExercisesHelpers() async { + await DebugSettings.exerciseForTesting() + } +} diff --git a/apps/macos/Tests/OpenClawIPCTests/MacGatewayChatTransportMappingTests.swift b/apps/macos/Tests/OpenClawIPCTests/MacGatewayChatTransportMappingTests.swift new file mode 100644 index 0000000000000000000000000000000000000000..046e47886c2dd3f4b1b0f94309ec2d9370f1e933 --- /dev/null +++ b/apps/macos/Tests/OpenClawIPCTests/MacGatewayChatTransportMappingTests.swift @@ -0,0 +1,99 @@ +import OpenClawChatUI +import OpenClawProtocol +import Testing +@testable import OpenClaw + +@Suite struct MacGatewayChatTransportMappingTests { + @Test func snapshotMapsToHealth() { + let snapshot = Snapshot( + presence: [], + health: OpenClawProtocol.AnyCodable(["ok": OpenClawProtocol.AnyCodable(false)]), + stateversion: StateVersion(presence: 1, health: 1), + uptimems: 123, + configpath: nil, + statedir: nil, + sessiondefaults: nil) + + let hello = HelloOk( + type: "hello", + _protocol: 2, + server: [:], + features: [:], + snapshot: snapshot, + canvashosturl: nil, + auth: nil, + policy: [:]) + + let mapped = MacGatewayChatTransport.mapPushToTransportEvent(.snapshot(hello)) + switch mapped { + case let .health(ok): + #expect(ok == false) + default: + Issue.record("expected .health from snapshot, got \(String(describing: mapped))") + } + } + + @Test func healthEventMapsToHealth() { + let frame = EventFrame( + type: "event", + event: "health", + payload: OpenClawProtocol.AnyCodable(["ok": OpenClawProtocol.AnyCodable(true)]), + seq: 1, + stateversion: nil) + + let mapped = MacGatewayChatTransport.mapPushToTransportEvent(.event(frame)) + switch mapped { + case let .health(ok): + #expect(ok == true) + default: + Issue.record("expected .health from health event, got \(String(describing: mapped))") + } + } + + @Test func tickEventMapsToTick() { + let frame = EventFrame(type: "event", event: "tick", payload: nil, seq: 1, stateversion: nil) + let mapped = MacGatewayChatTransport.mapPushToTransportEvent(.event(frame)) + #expect({ + if case .tick = mapped { return true } + return false + }()) + } + + @Test func chatEventMapsToChat() { + let payload = OpenClawProtocol.AnyCodable([ + "runId": OpenClawProtocol.AnyCodable("run-1"), + "sessionKey": OpenClawProtocol.AnyCodable("main"), + "state": OpenClawProtocol.AnyCodable("final"), + ]) + let frame = EventFrame(type: "event", event: "chat", payload: payload, seq: 1, stateversion: nil) + let mapped = MacGatewayChatTransport.mapPushToTransportEvent(.event(frame)) + + switch mapped { + case let .chat(chat): + #expect(chat.runId == "run-1") + #expect(chat.sessionKey == "main") + #expect(chat.state == "final") + default: + Issue.record("expected .chat from chat event, got \(String(describing: mapped))") + } + } + + @Test func unknownEventMapsToNil() { + let frame = EventFrame( + type: "event", + event: "unknown", + payload: OpenClawProtocol.AnyCodable(["a": OpenClawProtocol.AnyCodable(1)]), + seq: 1, + stateversion: nil) + let mapped = MacGatewayChatTransport.mapPushToTransportEvent(.event(frame)) + #expect(mapped == nil) + } + + @Test func seqGapMapsToSeqGap() { + let mapped = MacGatewayChatTransport.mapPushToTransportEvent(.seqGap(expected: 1, received: 9)) + #expect({ + if case .seqGap = mapped { return true } + return false + }()) + } +} diff --git a/apps/macos/Tests/OpenClawIPCTests/MacNodeRuntimeTests.swift b/apps/macos/Tests/OpenClawIPCTests/MacNodeRuntimeTests.swift new file mode 100644 index 0000000000000000000000000000000000000000..866256241a2cd9969ae6c08a623a426c1a151b35 --- /dev/null +++ b/apps/macos/Tests/OpenClawIPCTests/MacNodeRuntimeTests.swift @@ -0,0 +1,97 @@ +import OpenClawKit +import CoreLocation +import Foundation +import Testing +@testable import OpenClaw + +struct MacNodeRuntimeTests { + @Test func handleInvokeRejectsUnknownCommand() async { + let runtime = MacNodeRuntime() + let response = await runtime.handleInvoke( + BridgeInvokeRequest(id: "req-1", command: "unknown.command")) + #expect(response.ok == false) + } + + @Test func handleInvokeRejectsEmptySystemRun() async throws { + let runtime = MacNodeRuntime() + let params = OpenClawSystemRunParams(command: []) + let json = try String(data: JSONEncoder().encode(params), encoding: .utf8) + let response = await runtime.handleInvoke( + BridgeInvokeRequest(id: "req-2", command: OpenClawSystemCommand.run.rawValue, paramsJSON: json)) + #expect(response.ok == false) + } + + @Test func handleInvokeRejectsEmptySystemWhich() async throws { + let runtime = MacNodeRuntime() + let params = OpenClawSystemWhichParams(bins: []) + let json = try String(data: JSONEncoder().encode(params), encoding: .utf8) + let response = await runtime.handleInvoke( + BridgeInvokeRequest(id: "req-2b", command: OpenClawSystemCommand.which.rawValue, paramsJSON: json)) + #expect(response.ok == false) + } + + @Test func handleInvokeRejectsEmptyNotification() async throws { + let runtime = MacNodeRuntime() + let params = OpenClawSystemNotifyParams(title: "", body: "") + let json = try String(data: JSONEncoder().encode(params), encoding: .utf8) + let response = await runtime.handleInvoke( + BridgeInvokeRequest(id: "req-3", command: OpenClawSystemCommand.notify.rawValue, paramsJSON: json)) + #expect(response.ok == false) + } + + @Test func handleInvokeCameraListRequiresEnabledCamera() async { + await TestIsolation.withUserDefaultsValues([cameraEnabledKey: false]) { + let runtime = MacNodeRuntime() + let response = await runtime.handleInvoke( + BridgeInvokeRequest(id: "req-4", command: OpenClawCameraCommand.list.rawValue)) + #expect(response.ok == false) + #expect(response.error?.message.contains("CAMERA_DISABLED") == true) + } + } + + @Test func handleInvokeScreenRecordUsesInjectedServices() async throws { + @MainActor + final class FakeMainActorServices: MacNodeRuntimeMainActorServices, @unchecked Sendable { + func recordScreen( + screenIndex: Int?, + durationMs: Int?, + fps: Double?, + includeAudio: Bool?, + outPath: String?) async throws -> (path: String, hasAudio: Bool) + { + let url = FileManager().temporaryDirectory + .appendingPathComponent("openclaw-test-screen-record-\(UUID().uuidString).mp4") + try Data("ok".utf8).write(to: url) + return (path: url.path, hasAudio: false) + } + + func locationAuthorizationStatus() -> CLAuthorizationStatus { .authorizedAlways } + func locationAccuracyAuthorization() -> CLAccuracyAuthorization { .fullAccuracy } + func currentLocation( + desiredAccuracy: OpenClawLocationAccuracy, + maxAgeMs: Int?, + timeoutMs: Int?) async throws -> CLLocation + { + CLLocation(latitude: 0, longitude: 0) + } + } + + let services = await MainActor.run { FakeMainActorServices() } + let runtime = MacNodeRuntime(makeMainActorServices: { services }) + + let params = MacNodeScreenRecordParams(durationMs: 250) + let json = try String(data: JSONEncoder().encode(params), encoding: .utf8) + let response = await runtime.handleInvoke( + BridgeInvokeRequest(id: "req-5", command: MacNodeScreenCommand.record.rawValue, paramsJSON: json)) + #expect(response.ok == true) + let payloadJSON = try #require(response.payloadJSON) + + struct Payload: Decodable { + var format: String + var base64: String + } + let payload = try JSONDecoder().decode(Payload.self, from: Data(payloadJSON.utf8)) + #expect(payload.format == "mp4") + #expect(!payload.base64.isEmpty) + } +} diff --git a/apps/macos/Tests/OpenClawIPCTests/MasterDiscoveryMenuSmokeTests.swift b/apps/macos/Tests/OpenClawIPCTests/MasterDiscoveryMenuSmokeTests.swift new file mode 100644 index 0000000000000000000000000000000000000000..c6d58cc3a86b3546bd151e192acee02ba7de0f6e --- /dev/null +++ b/apps/macos/Tests/OpenClawIPCTests/MasterDiscoveryMenuSmokeTests.swift @@ -0,0 +1,78 @@ +import OpenClawDiscovery +import SwiftUI +import Testing +@testable import OpenClaw + +@Suite(.serialized) +@MainActor +struct MasterDiscoveryMenuSmokeTests { + @Test func inlineListBuildsBodyWhenEmpty() { + let discovery = GatewayDiscoveryModel(localDisplayName: InstanceIdentity.displayName) + discovery.statusText = "Searching…" + discovery.gateways = [] + + let view = GatewayDiscoveryInlineList( + discovery: discovery, + currentTarget: nil, + currentUrl: nil, + transport: .ssh, + onSelect: { _ in }) + _ = view.body + } + + @Test func inlineListBuildsBodyWithMasterAndSelection() { + let discovery = GatewayDiscoveryModel(localDisplayName: InstanceIdentity.displayName) + discovery.statusText = "Found 1" + discovery.gateways = [ + GatewayDiscoveryModel.DiscoveredGateway( + displayName: "Office Mac", + lanHost: "office.local", + tailnetDns: "office.tailnet-123.ts.net", + sshPort: 2222, + gatewayPort: nil, + cliPath: nil, + stableID: "office", + debugID: "office", + isLocal: false), + ] + + let currentTarget = "\(NSUserName())@office.tailnet-123.ts.net:2222" + let view = GatewayDiscoveryInlineList( + discovery: discovery, + currentTarget: currentTarget, + currentUrl: nil, + transport: .ssh, + onSelect: { _ in }) + _ = view.body + } + + @Test func menuBuildsBodyWithMasters() { + let discovery = GatewayDiscoveryModel(localDisplayName: InstanceIdentity.displayName) + discovery.statusText = "Found 2" + discovery.gateways = [ + GatewayDiscoveryModel.DiscoveredGateway( + displayName: "A", + lanHost: "a.local", + tailnetDns: nil, + sshPort: 22, + gatewayPort: nil, + cliPath: nil, + stableID: "a", + debugID: "a", + isLocal: false), + GatewayDiscoveryModel.DiscoveredGateway( + displayName: "B", + lanHost: nil, + tailnetDns: "b.ts.net", + sshPort: 22, + gatewayPort: nil, + cliPath: nil, + stableID: "b", + debugID: "b", + isLocal: false), + ] + + let view = GatewayDiscoveryMenu(discovery: discovery, onSelect: { _ in }) + _ = view.body + } +} diff --git a/apps/macos/Tests/OpenClawIPCTests/MenuContentSmokeTests.swift b/apps/macos/Tests/OpenClawIPCTests/MenuContentSmokeTests.swift new file mode 100644 index 0000000000000000000000000000000000000000..a57782148e47c30797450859865e7d5716e801fd --- /dev/null +++ b/apps/macos/Tests/OpenClawIPCTests/MenuContentSmokeTests.swift @@ -0,0 +1,41 @@ +import SwiftUI +import Testing +@testable import OpenClaw + +@Suite(.serialized) +@MainActor +struct MenuContentSmokeTests { + @Test func menuContentBuildsBodyLocalMode() { + let state = AppState(preview: true) + state.connectionMode = .local + let view = MenuContent(state: state, updater: nil) + _ = view.body + } + + @Test func menuContentBuildsBodyRemoteMode() { + let state = AppState(preview: true) + state.connectionMode = .remote + let view = MenuContent(state: state, updater: nil) + _ = view.body + } + + @Test func menuContentBuildsBodyUnconfiguredMode() { + let state = AppState(preview: true) + state.connectionMode = .unconfigured + let view = MenuContent(state: state, updater: nil) + _ = view.body + } + + @Test func menuContentBuildsBodyWithDebugAndCanvas() { + let state = AppState(preview: true) + state.connectionMode = .local + state.debugPaneEnabled = true + state.canvasEnabled = true + state.canvasPanelVisible = true + state.swabbleEnabled = true + state.voicePushToTalkEnabled = true + state.heartbeatsEnabled = true + let view = MenuContent(state: state, updater: nil) + _ = view.body + } +} diff --git a/apps/macos/Tests/OpenClawIPCTests/MenuSessionsInjectorTests.swift b/apps/macos/Tests/OpenClawIPCTests/MenuSessionsInjectorTests.swift new file mode 100644 index 0000000000000000000000000000000000000000..0228101f57b99c5a8d709af3c8d399d8bc817bf6 --- /dev/null +++ b/apps/macos/Tests/OpenClawIPCTests/MenuSessionsInjectorTests.swift @@ -0,0 +1,96 @@ +import AppKit +import Testing +@testable import OpenClaw + +@Suite(.serialized) +@MainActor +struct MenuSessionsInjectorTests { + @Test func injectsDisconnectedMessage() { + let injector = MenuSessionsInjector() + injector.setTestingControlChannelConnected(false) + injector.setTestingSnapshot(nil, errorText: nil) + + let menu = NSMenu() + menu.addItem(NSMenuItem(title: "Header", action: nil, keyEquivalent: "")) + menu.addItem(.separator()) + menu.addItem(NSMenuItem(title: "Send Heartbeats", action: nil, keyEquivalent: "")) + + injector.injectForTesting(into: menu) + #expect(menu.items.contains { $0.tag == 9_415_557 }) + } + + @Test func injectsSessionRows() { + let injector = MenuSessionsInjector() + injector.setTestingControlChannelConnected(true) + + let defaults = SessionDefaults(model: "anthropic/claude-opus-4-5", contextTokens: 200_000) + let rows = [ + SessionRow( + id: "main", + key: "main", + kind: .direct, + displayName: nil, + provider: nil, + subject: nil, + room: nil, + space: nil, + updatedAt: Date(), + sessionId: "s1", + thinkingLevel: "low", + verboseLevel: nil, + systemSent: false, + abortedLastRun: false, + tokens: SessionTokenStats(input: 10, output: 20, total: 30, contextTokens: 200_000), + model: "claude-opus-4-5"), + SessionRow( + id: "discord:group:alpha", + key: "discord:group:alpha", + kind: .group, + displayName: nil, + provider: nil, + subject: nil, + room: nil, + space: nil, + updatedAt: Date(timeIntervalSinceNow: -60), + sessionId: "s2", + thinkingLevel: "high", + verboseLevel: "debug", + systemSent: true, + abortedLastRun: true, + tokens: SessionTokenStats(input: 50, output: 50, total: 100, contextTokens: 200_000), + model: "claude-opus-4-5"), + ] + let snapshot = SessionStoreSnapshot( + storePath: "/tmp/sessions.json", + defaults: defaults, + rows: rows) + injector.setTestingSnapshot(snapshot, errorText: nil) + + let usage = GatewayUsageSummary( + updatedAt: Date().timeIntervalSince1970 * 1000, + providers: [ + GatewayUsageProvider( + provider: "anthropic", + displayName: "Claude", + windows: [GatewayUsageWindow(label: "5h", usedPercent: 12, resetAt: nil)], + plan: "Pro", + error: nil), + GatewayUsageProvider( + provider: "openai-codex", + displayName: "Codex", + windows: [GatewayUsageWindow(label: "day", usedPercent: 3, resetAt: nil)], + plan: nil, + error: nil), + ]) + injector.setTestingUsageSummary(usage, errorText: nil) + + let menu = NSMenu() + menu.addItem(NSMenuItem(title: "Header", action: nil, keyEquivalent: "")) + menu.addItem(.separator()) + menu.addItem(NSMenuItem(title: "Send Heartbeats", action: nil, keyEquivalent: "")) + + injector.injectForTesting(into: menu) + #expect(menu.items.contains { $0.tag == 9_415_557 }) + #expect(menu.items.contains { $0.tag == 9_415_557 && $0.isSeparatorItem }) + } +} diff --git a/apps/macos/Tests/OpenClawIPCTests/ModelCatalogLoaderTests.swift b/apps/macos/Tests/OpenClawIPCTests/ModelCatalogLoaderTests.swift new file mode 100644 index 0000000000000000000000000000000000000000..05ed6f8513bbcb05b7572321ca971c847207055a --- /dev/null +++ b/apps/macos/Tests/OpenClawIPCTests/ModelCatalogLoaderTests.swift @@ -0,0 +1,53 @@ +import Foundation +import Testing +@testable import OpenClaw + +@Suite +struct ModelCatalogLoaderTests { + @Test + func loadParsesModelsFromTypeScriptAndSorts() async throws { + let src = """ + export const MODELS = { + openai: { + "gpt-4o-mini": { name: "GPT-4o mini", contextWindow: 128000 } satisfies any, + "gpt-4o": { name: "GPT-4o", contextWindow: 128000 } as any, + "gpt-3.5": { contextWindow: 16000 }, + }, + anthropic: { + "claude-3": { name: "Claude 3", contextWindow: 200000 }, + }, + }; + """ + + let tmp = FileManager().temporaryDirectory + .appendingPathComponent("models-\(UUID().uuidString).ts") + defer { try? FileManager().removeItem(at: tmp) } + try src.write(to: tmp, atomically: true, encoding: .utf8) + + let choices = try await ModelCatalogLoader.load(from: tmp.path) + #expect(choices.count == 4) + #expect(choices.first?.provider == "anthropic") + #expect(choices.first?.id == "claude-3") + + let ids = Set(choices.map(\.id)) + #expect(ids == Set(["claude-3", "gpt-4o", "gpt-4o-mini", "gpt-3.5"])) + + let openai = choices.filter { $0.provider == "openai" } + let openaiNames = openai.map(\.name) + #expect(openaiNames == openaiNames.sorted { a, b in + a.localizedCaseInsensitiveCompare(b) == .orderedAscending + }) + } + + @Test + func loadWithNoExportReturnsEmptyChoices() async throws { + let src = "const NOPE = 1;" + let tmp = FileManager().temporaryDirectory + .appendingPathComponent("models-\(UUID().uuidString).ts") + defer { try? FileManager().removeItem(at: tmp) } + try src.write(to: tmp, atomically: true, encoding: .utf8) + + let choices = try await ModelCatalogLoader.load(from: tmp.path) + #expect(choices.isEmpty) + } +} diff --git a/apps/macos/Tests/OpenClawIPCTests/NodeManagerPathsTests.swift b/apps/macos/Tests/OpenClawIPCTests/NodeManagerPathsTests.swift new file mode 100644 index 0000000000000000000000000000000000000000..9ee41b4f7b986192f29de28e671b5fa4b71f419b --- /dev/null +++ b/apps/macos/Tests/OpenClawIPCTests/NodeManagerPathsTests.swift @@ -0,0 +1,45 @@ +import Foundation +import Testing +@testable import OpenClaw + +@Suite struct NodeManagerPathsTests { + private func makeTempDir() throws -> URL { + let base = URL(fileURLWithPath: NSTemporaryDirectory(), isDirectory: true) + let dir = base.appendingPathComponent(UUID().uuidString, isDirectory: true) + try FileManager().createDirectory(at: dir, withIntermediateDirectories: true) + return dir + } + + private func makeExec(at path: URL) throws { + try FileManager().createDirectory( + at: path.deletingLastPathComponent(), + withIntermediateDirectories: true) + FileManager().createFile(atPath: path.path, contents: Data("echo ok\n".utf8)) + try FileManager().setAttributes([.posixPermissions: 0o755], ofItemAtPath: path.path) + } + + @Test func fnmNodeBinsPreferNewestInstalledVersion() throws { + let home = try self.makeTempDir() + + let v20Bin = home + .appendingPathComponent(".local/share/fnm/node-versions/v20.19.5/installation/bin/node") + let v25Bin = home + .appendingPathComponent(".local/share/fnm/node-versions/v25.1.0/installation/bin/node") + try self.makeExec(at: v20Bin) + try self.makeExec(at: v25Bin) + + let bins = CommandResolver._testNodeManagerBinPaths(home: home) + #expect(bins.first == v25Bin.deletingLastPathComponent().path) + #expect(bins.contains(v20Bin.deletingLastPathComponent().path)) + } + + @Test func ignoresEntriesWithoutNodeExecutable() throws { + let home = try self.makeTempDir() + let missingNodeBin = home + .appendingPathComponent(".local/share/fnm/node-versions/v99.0.0/installation/bin") + try FileManager().createDirectory(at: missingNodeBin, withIntermediateDirectories: true) + + let bins = CommandResolver._testNodeManagerBinPaths(home: home) + #expect(!bins.contains(missingNodeBin.path)) + } +} diff --git a/apps/macos/Tests/OpenClawIPCTests/NodePairingApprovalPrompterTests.swift b/apps/macos/Tests/OpenClawIPCTests/NodePairingApprovalPrompterTests.swift new file mode 100644 index 0000000000000000000000000000000000000000..7c2a90e456ec7dbda4260e2666b6eb171bbba8d0 --- /dev/null +++ b/apps/macos/Tests/OpenClawIPCTests/NodePairingApprovalPrompterTests.swift @@ -0,0 +1,10 @@ +import Testing +@testable import OpenClaw + +@Suite(.serialized) +@MainActor +struct NodePairingApprovalPrompterTests { + @Test func nodePairingApprovalPrompterExercises() async { + await NodePairingApprovalPrompter.exerciseForTesting() + } +} diff --git a/apps/macos/Tests/OpenClawIPCTests/NodePairingReconcilePolicyTests.swift b/apps/macos/Tests/OpenClawIPCTests/NodePairingReconcilePolicyTests.swift new file mode 100644 index 0000000000000000000000000000000000000000..cc1113f789cde3c0a86446eb3f40534f958f0337 --- /dev/null +++ b/apps/macos/Tests/OpenClawIPCTests/NodePairingReconcilePolicyTests.swift @@ -0,0 +1,14 @@ +import Testing +@testable import OpenClaw + +@Suite struct NodePairingReconcilePolicyTests { + @Test func policyPollsOnlyWhenActive() { + #expect(NodePairingReconcilePolicy.shouldPoll(pendingCount: 0, isPresenting: false) == false) + #expect(NodePairingReconcilePolicy.shouldPoll(pendingCount: 1, isPresenting: false)) + #expect(NodePairingReconcilePolicy.shouldPoll(pendingCount: 0, isPresenting: true)) + } + + @Test func policyUsesSlowSafetyInterval() { + #expect(NodePairingReconcilePolicy.activeIntervalMs >= 10000) + } +} diff --git a/apps/macos/Tests/OpenClawIPCTests/OnboardingCoverageTests.swift b/apps/macos/Tests/OpenClawIPCTests/OnboardingCoverageTests.swift new file mode 100644 index 0000000000000000000000000000000000000000..e79d002683c83dd2abe2303db8c95208f1ade4d1 --- /dev/null +++ b/apps/macos/Tests/OpenClawIPCTests/OnboardingCoverageTests.swift @@ -0,0 +1,10 @@ +import Testing +@testable import OpenClaw + +@Suite(.serialized) +@MainActor +struct OnboardingCoverageTests { + @Test func exerciseOnboardingPages() { + OnboardingView.exerciseForTesting() + } +} diff --git a/apps/macos/Tests/OpenClawIPCTests/OnboardingViewSmokeTests.swift b/apps/macos/Tests/OpenClawIPCTests/OnboardingViewSmokeTests.swift new file mode 100644 index 0000000000000000000000000000000000000000..57912eb412d1e97c8f05d6721f843313a5230bb8 --- /dev/null +++ b/apps/macos/Tests/OpenClawIPCTests/OnboardingViewSmokeTests.swift @@ -0,0 +1,28 @@ +import OpenClawDiscovery +import SwiftUI +import Testing +@testable import OpenClaw + +@Suite(.serialized) +@MainActor +struct OnboardingViewSmokeTests { + @Test func onboardingViewBuildsBody() { + let state = AppState(preview: true) + let view = OnboardingView( + state: state, + permissionMonitor: PermissionMonitor.shared, + discoveryModel: GatewayDiscoveryModel(localDisplayName: InstanceIdentity.displayName)) + _ = view.body + } + + @Test func pageOrderOmitsWorkspaceAndIdentitySteps() { + let order = OnboardingView.pageOrder(for: .local, showOnboardingChat: false) + #expect(!order.contains(7)) + #expect(order.contains(3)) + } + + @Test func pageOrderOmitsOnboardingChatWhenIdentityKnown() { + let order = OnboardingView.pageOrder(for: .local, showOnboardingChat: false) + #expect(!order.contains(8)) + } +} diff --git a/apps/macos/Tests/OpenClawIPCTests/OnboardingWizardStepViewTests.swift b/apps/macos/Tests/OpenClawIPCTests/OnboardingWizardStepViewTests.swift new file mode 100644 index 0000000000000000000000000000000000000000..7211482fea227cb84bc6922568827a52a86c9f8d --- /dev/null +++ b/apps/macos/Tests/OpenClawIPCTests/OnboardingWizardStepViewTests.swift @@ -0,0 +1,44 @@ +import OpenClawProtocol +import SwiftUI +import Testing +@testable import OpenClaw + +private typealias ProtoAnyCodable = OpenClawProtocol.AnyCodable + +@Suite(.serialized) +@MainActor +struct OnboardingWizardStepViewTests { + @Test func noteStepBuilds() { + let step = WizardStep( + id: "step-1", + type: ProtoAnyCodable("note"), + title: "Welcome", + message: "Hello", + options: nil, + initialvalue: nil, + placeholder: nil, + sensitive: nil, + executor: nil) + let view = OnboardingWizardStepView(step: step, isSubmitting: false, onSubmit: { _ in }) + _ = view.body + } + + @Test func selectStepBuilds() { + let options: [[String: ProtoAnyCodable]] = [ + ["value": ProtoAnyCodable("local"), "label": ProtoAnyCodable("Local"), "hint": ProtoAnyCodable("This Mac")], + ["value": ProtoAnyCodable("remote"), "label": ProtoAnyCodable("Remote")], + ] + let step = WizardStep( + id: "step-2", + type: ProtoAnyCodable("select"), + title: "Mode", + message: "Choose a mode", + options: options, + initialvalue: ProtoAnyCodable("local"), + placeholder: nil, + sensitive: nil, + executor: nil) + let view = OnboardingWizardStepView(step: step, isSubmitting: false, onSubmit: { _ in }) + _ = view.body + } +} diff --git a/apps/macos/Tests/OpenClawIPCTests/OpenClawConfigFileTests.swift b/apps/macos/Tests/OpenClawIPCTests/OpenClawConfigFileTests.swift new file mode 100644 index 0000000000000000000000000000000000000000..c03505e2f4cd03cf10286591b44494f04fcdeccf --- /dev/null +++ b/apps/macos/Tests/OpenClawIPCTests/OpenClawConfigFileTests.swift @@ -0,0 +1,79 @@ +import Foundation +import Testing +@testable import OpenClaw + +@Suite(.serialized) +struct OpenClawConfigFileTests { + @Test + func configPathRespectsEnvOverride() async { + let override = FileManager().temporaryDirectory + .appendingPathComponent("openclaw-config-\(UUID().uuidString)") + .appendingPathComponent("openclaw.json") + .path + + await TestIsolation.withEnvValues(["OPENCLAW_CONFIG_PATH": override]) { + #expect(OpenClawConfigFile.url().path == override) + } + } + + @MainActor + @Test + func remoteGatewayPortParsesAndMatchesHost() async { + let override = FileManager().temporaryDirectory + .appendingPathComponent("openclaw-config-\(UUID().uuidString)") + .appendingPathComponent("openclaw.json") + .path + + await TestIsolation.withEnvValues(["OPENCLAW_CONFIG_PATH": override]) { + OpenClawConfigFile.saveDict([ + "gateway": [ + "remote": [ + "url": "ws://gateway.ts.net:19999", + ], + ], + ]) + #expect(OpenClawConfigFile.remoteGatewayPort() == 19999) + #expect(OpenClawConfigFile.remoteGatewayPort(matchingHost: "gateway.ts.net") == 19999) + #expect(OpenClawConfigFile.remoteGatewayPort(matchingHost: "gateway") == 19999) + #expect(OpenClawConfigFile.remoteGatewayPort(matchingHost: "other.ts.net") == nil) + } + } + + @MainActor + @Test + func setRemoteGatewayUrlPreservesScheme() async { + let override = FileManager().temporaryDirectory + .appendingPathComponent("openclaw-config-\(UUID().uuidString)") + .appendingPathComponent("openclaw.json") + .path + + await TestIsolation.withEnvValues(["OPENCLAW_CONFIG_PATH": override]) { + OpenClawConfigFile.saveDict([ + "gateway": [ + "remote": [ + "url": "wss://old-host:111", + ], + ], + ]) + OpenClawConfigFile.setRemoteGatewayUrl(host: "new-host", port: 2222) + let root = OpenClawConfigFile.loadDict() + let url = ((root["gateway"] as? [String: Any])?["remote"] as? [String: Any])?["url"] as? String + #expect(url == "wss://new-host:2222") + } + } + + @Test + func stateDirOverrideSetsConfigPath() async { + let dir = FileManager().temporaryDirectory + .appendingPathComponent("openclaw-state-\(UUID().uuidString)", isDirectory: true) + .path + + await TestIsolation.withEnvValues([ + "OPENCLAW_CONFIG_PATH": nil, + "OPENCLAW_STATE_DIR": dir, + ]) { + #expect(OpenClawConfigFile.stateDirURL().path == dir) + #expect(OpenClawConfigFile.url().path == "\(dir)/openclaw.json") + } + } +} diff --git a/apps/macos/Tests/OpenClawIPCTests/OpenClawOAuthStoreTests.swift b/apps/macos/Tests/OpenClawIPCTests/OpenClawOAuthStoreTests.swift new file mode 100644 index 0000000000000000000000000000000000000000..b34e9c3008abd19d1dd9c95d3c70b77c006642c4 --- /dev/null +++ b/apps/macos/Tests/OpenClawIPCTests/OpenClawOAuthStoreTests.swift @@ -0,0 +1,97 @@ +import Foundation +import Testing +@testable import OpenClaw + +@Suite +struct OpenClawOAuthStoreTests { + @Test + func returnsMissingWhenFileAbsent() { + let url = FileManager().temporaryDirectory + .appendingPathComponent("openclaw-oauth-\(UUID().uuidString)") + .appendingPathComponent("oauth.json") + #expect(OpenClawOAuthStore.anthropicOAuthStatus(at: url) == .missingFile) + } + + @Test + func usesEnvOverrideForOpenClawOAuthDir() throws { + let key = "OPENCLAW_OAUTH_DIR" + let previous = ProcessInfo.processInfo.environment[key] + defer { + if let previous { + setenv(key, previous, 1) + } else { + unsetenv(key) + } + } + + let dir = FileManager().temporaryDirectory + .appendingPathComponent("openclaw-oauth-\(UUID().uuidString)", isDirectory: true) + setenv(key, dir.path, 1) + + #expect(OpenClawOAuthStore.oauthDir().standardizedFileURL == dir.standardizedFileURL) + } + + @Test + func acceptsPiFormatTokens() throws { + let url = try self.writeOAuthFile([ + "anthropic": [ + "type": "oauth", + "refresh": "r1", + "access": "a1", + "expires": 1_234_567_890, + ], + ]) + + #expect(OpenClawOAuthStore.anthropicOAuthStatus(at: url).isConnected) + } + + @Test + func acceptsTokenKeyVariants() throws { + let url = try self.writeOAuthFile([ + "anthropic": [ + "type": "oauth", + "refresh_token": "r1", + "access_token": "a1", + ], + ]) + + #expect(OpenClawOAuthStore.anthropicOAuthStatus(at: url).isConnected) + } + + @Test + func reportsMissingProviderEntry() throws { + let url = try self.writeOAuthFile([ + "other": [ + "type": "oauth", + "refresh": "r1", + "access": "a1", + ], + ]) + + #expect(OpenClawOAuthStore.anthropicOAuthStatus(at: url) == .missingProviderEntry) + } + + @Test + func reportsMissingTokens() throws { + let url = try self.writeOAuthFile([ + "anthropic": [ + "type": "oauth", + "refresh": "", + "access": "a1", + ], + ]) + + #expect(OpenClawOAuthStore.anthropicOAuthStatus(at: url) == .missingTokens) + } + + private func writeOAuthFile(_ json: [String: Any]) throws -> URL { + let dir = FileManager().temporaryDirectory + .appendingPathComponent("openclaw-oauth-\(UUID().uuidString)", isDirectory: true) + try FileManager().createDirectory(at: dir, withIntermediateDirectories: true) + + let url = dir.appendingPathComponent("oauth.json") + let data = try JSONSerialization.data(withJSONObject: json, options: [.prettyPrinted, .sortedKeys]) + try data.write(to: url, options: [.atomic]) + return url + } +} diff --git a/apps/macos/Tests/OpenClawIPCTests/PermissionManagerLocationTests.swift b/apps/macos/Tests/OpenClawIPCTests/PermissionManagerLocationTests.swift new file mode 100644 index 0000000000000000000000000000000000000000..871998cb2404be73a208f50c4e59b8a05873facb --- /dev/null +++ b/apps/macos/Tests/OpenClawIPCTests/PermissionManagerLocationTests.swift @@ -0,0 +1,20 @@ +import CoreLocation +import Testing + +@testable import OpenClaw + +@Suite("PermissionManager Location") +struct PermissionManagerLocationTests { + @Test("authorizedAlways counts for both modes") + func authorizedAlwaysCountsForBothModes() { + #expect(PermissionManager.isLocationAuthorized(status: .authorizedAlways, requireAlways: false)) + #expect(PermissionManager.isLocationAuthorized(status: .authorizedAlways, requireAlways: true)) + } + + @Test("other statuses not authorized") + func otherStatusesNotAuthorized() { + #expect(!PermissionManager.isLocationAuthorized(status: .notDetermined, requireAlways: false)) + #expect(!PermissionManager.isLocationAuthorized(status: .denied, requireAlways: false)) + #expect(!PermissionManager.isLocationAuthorized(status: .restricted, requireAlways: false)) + } +} diff --git a/apps/macos/Tests/OpenClawIPCTests/PermissionManagerTests.swift b/apps/macos/Tests/OpenClawIPCTests/PermissionManagerTests.swift new file mode 100644 index 0000000000000000000000000000000000000000..5e41339f166ef50206a801180cbefcbd6a30d00d --- /dev/null +++ b/apps/macos/Tests/OpenClawIPCTests/PermissionManagerTests.swift @@ -0,0 +1,38 @@ +import OpenClawIPC +import CoreLocation +import Testing +@testable import OpenClaw + +@Suite(.serialized) +@MainActor +struct PermissionManagerTests { + @Test func voiceWakePermissionHelpersMatchStatus() async { + let direct = PermissionManager.voiceWakePermissionsGranted() + let ensured = await PermissionManager.ensureVoiceWakePermissions(interactive: false) + #expect(ensured == direct) + } + + @Test func statusCanQueryNonInteractiveCaps() async { + let caps: [Capability] = [.microphone, .speechRecognition, .screenRecording] + let status = await PermissionManager.status(caps) + #expect(status.keys.count == caps.count) + } + + @Test func ensureNonInteractiveDoesNotThrow() async { + let caps: [Capability] = [.microphone, .speechRecognition, .screenRecording] + let ensured = await PermissionManager.ensure(caps, interactive: false) + #expect(ensured.keys.count == caps.count) + } + + @Test func locationStatusMatchesAuthorizationAlways() async { + let status = CLLocationManager().authorizationStatus + let results = await PermissionManager.status([.location]) + #expect(results[.location] == (status == .authorizedAlways)) + } + + @Test func ensureLocationNonInteractiveMatchesAuthorizationAlways() async { + let status = CLLocationManager().authorizationStatus + let ensured = await PermissionManager.ensure([.location], interactive: false) + #expect(ensured[.location] == (status == .authorizedAlways)) + } +} diff --git a/apps/macos/Tests/OpenClawIPCTests/Placeholder.swift b/apps/macos/Tests/OpenClawIPCTests/Placeholder.swift new file mode 100644 index 0000000000000000000000000000000000000000..14e5c056b0971addd8f183161321feaec67a69e2 --- /dev/null +++ b/apps/macos/Tests/OpenClawIPCTests/Placeholder.swift @@ -0,0 +1,7 @@ +import Testing + +@Suite struct PlaceholderTests { + @Test func placeholder() { + #expect(true) + } +} diff --git a/apps/macos/Tests/OpenClawIPCTests/RemotePortTunnelTests.swift b/apps/macos/Tests/OpenClawIPCTests/RemotePortTunnelTests.swift new file mode 100644 index 0000000000000000000000000000000000000000..856af89676c01a1e92f6193376539d30c9422916 --- /dev/null +++ b/apps/macos/Tests/OpenClawIPCTests/RemotePortTunnelTests.swift @@ -0,0 +1,74 @@ +import Testing +@testable import OpenClaw + +#if canImport(Darwin) +import Darwin +import Foundation + +@Suite struct RemotePortTunnelTests { + @Test func drainStderrDoesNotCrashWhenHandleClosed() { + let pipe = Pipe() + let handle = pipe.fileHandleForReading + try? handle.close() + + let drained = RemotePortTunnel._testDrainStderr(handle) + #expect(drained.isEmpty) + } + + @Test func portIsFreeDetectsIPv4Listener() { + var fd = socket(AF_INET, SOCK_STREAM, 0) + #expect(fd >= 0) + guard fd >= 0 else { return } + defer { + if fd >= 0 { _ = Darwin.close(fd) } + } + + var one: Int32 = 1 + _ = setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &one, socklen_t(MemoryLayout.size(ofValue: one))) + + var addr = sockaddr_in() + addr.sin_len = UInt8(MemoryLayout.size) + addr.sin_family = sa_family_t(AF_INET) + addr.sin_port = 0 + addr.sin_addr = in_addr(s_addr: inet_addr("127.0.0.1")) + + let bound = withUnsafePointer(to: &addr) { ptr in + ptr.withMemoryRebound(to: sockaddr.self, capacity: 1) { sa in + Darwin.bind(fd, sa, socklen_t(MemoryLayout.size)) + } + } + #expect(bound == 0) + guard bound == 0 else { return } + #expect(Darwin.listen(fd, 1) == 0) + + var name = sockaddr_in() + var nameLen = socklen_t(MemoryLayout.size) + let got = withUnsafeMutablePointer(to: &name) { ptr in + ptr.withMemoryRebound(to: sockaddr.self, capacity: 1) { sa in + getsockname(fd, sa, &nameLen) + } + } + #expect(got == 0) + guard got == 0 else { return } + + let port = UInt16(bigEndian: name.sin_port) + #expect(RemotePortTunnel._testPortIsFree(port) == false) + + _ = Darwin.close(fd) + fd = -1 + + // In parallel test runs, another test may briefly grab the same ephemeral port. + // Poll for a short window to avoid flakiness. + let deadline = Date().addingTimeInterval(0.5) + var free = false + while Date() < deadline { + if RemotePortTunnel._testPortIsFree(port) { + free = true + break + } + usleep(10000) // 10ms + } + #expect(free == true) + } +} +#endif diff --git a/apps/macos/Tests/OpenClawIPCTests/RuntimeLocatorTests.swift b/apps/macos/Tests/OpenClawIPCTests/RuntimeLocatorTests.swift new file mode 100644 index 0000000000000000000000000000000000000000..6662132c9ac726c5bf7efb056e13dbf331c0e4a0 --- /dev/null +++ b/apps/macos/Tests/OpenClawIPCTests/RuntimeLocatorTests.swift @@ -0,0 +1,71 @@ +import Foundation +import Testing +@testable import OpenClaw + +@Suite struct RuntimeLocatorTests { + private func makeTempExecutable(contents: String) throws -> URL { + let dir = URL(fileURLWithPath: NSTemporaryDirectory(), isDirectory: true) + .appendingPathComponent(UUID().uuidString, isDirectory: true) + try FileManager().createDirectory(at: dir, withIntermediateDirectories: true) + let path = dir.appendingPathComponent("node") + try contents.write(to: path, atomically: true, encoding: .utf8) + try FileManager().setAttributes([.posixPermissions: 0o755], ofItemAtPath: path.path) + return path + } + + @Test func resolveSucceedsWithValidNode() throws { + let script = """ + #!/bin/sh + echo v22.5.0 + """ + let node = try self.makeTempExecutable(contents: script) + let result = RuntimeLocator.resolve(searchPaths: [node.deletingLastPathComponent().path]) + guard case let .success(res) = result else { + Issue.record("Expected success, got \(result)") + return + } + #expect(res.path == node.path) + #expect(res.version == RuntimeVersion(major: 22, minor: 5, patch: 0)) + } + + @Test func resolveFailsWhenTooOld() throws { + let script = """ + #!/bin/sh + echo v18.2.0 + """ + let node = try self.makeTempExecutable(contents: script) + let result = RuntimeLocator.resolve(searchPaths: [node.deletingLastPathComponent().path]) + guard case let .failure(.unsupported(_, found, _, path, _)) = result else { + Issue.record("Expected unsupported error, got \(result)") + return + } + #expect(found == RuntimeVersion(major: 18, minor: 2, patch: 0)) + #expect(path == node.path) + } + + @Test func resolveFailsWhenVersionUnparsable() throws { + let script = """ + #!/bin/sh + echo node-version:unknown + """ + let node = try self.makeTempExecutable(contents: script) + let result = RuntimeLocator.resolve(searchPaths: [node.deletingLastPathComponent().path]) + guard case let .failure(.versionParse(_, raw, path, _)) = result else { + Issue.record("Expected versionParse error, got \(result)") + return + } + #expect(raw.contains("unknown")) + #expect(path == node.path) + } + + @Test func describeFailureIncludesPaths() { + let msg = RuntimeLocator.describeFailure(.notFound(searchPaths: ["/tmp/a", "/tmp/b"])) + #expect(msg.contains("PATH searched: /tmp/a:/tmp/b")) + } + + @Test func runtimeVersionParsesWithLeadingVAndMetadata() { + #expect(RuntimeVersion.from(string: "v22.1.3") == RuntimeVersion(major: 22, minor: 1, patch: 3)) + #expect(RuntimeVersion.from(string: "node 22.3.0-alpha.1") == RuntimeVersion(major: 22, minor: 3, patch: 0)) + #expect(RuntimeVersion.from(string: "bogus") == nil) + } +} diff --git a/apps/macos/Tests/OpenClawIPCTests/ScreenshotSizeTests.swift b/apps/macos/Tests/OpenClawIPCTests/ScreenshotSizeTests.swift new file mode 100644 index 0000000000000000000000000000000000000000..84fe17751dd5f513ece6af21ad185f1eff08561d --- /dev/null +++ b/apps/macos/Tests/OpenClawIPCTests/ScreenshotSizeTests.swift @@ -0,0 +1,21 @@ +import Foundation +import Testing +@testable import OpenClaw + +@Suite +struct ScreenshotSizeTests { + @Test + func readPNGSizeReturnsDimensions() throws { + let pngBase64 = + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+WZxkAAAAASUVORK5CYII=" + let data = try #require(Data(base64Encoded: pngBase64)) + let size = ScreenshotSize.readPNGSize(data: data) + #expect(size?.width == 1) + #expect(size?.height == 1) + } + + @Test + func readPNGSizeRejectsNonPNGData() { + #expect(ScreenshotSize.readPNGSize(data: Data("nope".utf8)) == nil) + } +} diff --git a/apps/macos/Tests/OpenClawIPCTests/SemverTests.swift b/apps/macos/Tests/OpenClawIPCTests/SemverTests.swift new file mode 100644 index 0000000000000000000000000000000000000000..83d8e8478f90623f34e489cc7c13ba5ce98cb0f7 --- /dev/null +++ b/apps/macos/Tests/OpenClawIPCTests/SemverTests.swift @@ -0,0 +1,21 @@ +import Testing +@testable import OpenClaw + +@Suite struct SemverTests { + @Test func comparisonOrdersByMajorMinorPatch() { + let a = Semver(major: 1, minor: 0, patch: 0) + let b = Semver(major: 1, minor: 1, patch: 0) + let c = Semver(major: 1, minor: 1, patch: 1) + let d = Semver(major: 2, minor: 0, patch: 0) + + #expect(a < b) + #expect(b < c) + #expect(c < d) + #expect(d > a) + } + + @Test func descriptionMatchesParts() { + let v = Semver(major: 3, minor: 2, patch: 1) + #expect(v.description == "3.2.1") + } +} diff --git a/apps/macos/Tests/OpenClawIPCTests/SessionDataTests.swift b/apps/macos/Tests/OpenClawIPCTests/SessionDataTests.swift new file mode 100644 index 0000000000000000000000000000000000000000..f1594ba7b543f1a1f8bd2b4f5cf84f685291cd89 --- /dev/null +++ b/apps/macos/Tests/OpenClawIPCTests/SessionDataTests.swift @@ -0,0 +1,48 @@ +import Foundation +import Testing +@testable import OpenClaw + +@Suite +struct SessionDataTests { + @Test func sessionKindFromKeyDetectsCommonKinds() { + #expect(SessionKind.from(key: "global") == .global) + #expect(SessionKind.from(key: "discord:group:engineering") == .group) + #expect(SessionKind.from(key: "unknown") == .unknown) + #expect(SessionKind.from(key: "user@example.com") == .direct) + } + + @Test func sessionTokenStatsFormatKTokensRoundsAsExpected() { + #expect(SessionTokenStats.formatKTokens(999) == "999") + #expect(SessionTokenStats.formatKTokens(1000) == "1.0k") + #expect(SessionTokenStats.formatKTokens(12340) == "12k") + } + + @Test func sessionTokenStatsPercentUsedClampsTo100() { + let stats = SessionTokenStats(input: 0, output: 0, total: 250_000, contextTokens: 200_000) + #expect(stats.percentUsed == 100) + } + + @Test func sessionRowFlagLabelsIncludeNonDefaultFlags() { + let row = SessionRow( + id: "x", + key: "user@example.com", + kind: .direct, + displayName: nil, + provider: nil, + subject: nil, + room: nil, + space: nil, + updatedAt: Date(), + sessionId: nil, + thinkingLevel: "high", + verboseLevel: "debug", + systemSent: true, + abortedLastRun: true, + tokens: SessionTokenStats(input: 1, output: 2, total: 3, contextTokens: 10), + model: nil) + #expect(row.flagLabels.contains("think high")) + #expect(row.flagLabels.contains("verbose debug")) + #expect(row.flagLabels.contains("system sent")) + #expect(row.flagLabels.contains("aborted")) + } +} diff --git a/apps/macos/Tests/OpenClawIPCTests/SessionMenuPreviewTests.swift b/apps/macos/Tests/OpenClawIPCTests/SessionMenuPreviewTests.swift new file mode 100644 index 0000000000000000000000000000000000000000..44bb3c39c2cf4cac62acc5ee22cde88bb4e65751 --- /dev/null +++ b/apps/macos/Tests/OpenClawIPCTests/SessionMenuPreviewTests.swift @@ -0,0 +1,28 @@ +import Foundation +import Testing +@testable import OpenClaw + +@Suite(.serialized) +struct SessionMenuPreviewTests { + @Test func loaderReturnsCachedItems() async { + await SessionPreviewCache.shared._testReset() + let items = [SessionPreviewItem(id: "1", role: .user, text: "Hi")] + let snapshot = SessionMenuPreviewSnapshot(items: items, status: .ready) + await SessionPreviewCache.shared._testSet(snapshot: snapshot, for: "main") + + let loaded = await SessionMenuPreviewLoader.load(sessionKey: "main", maxItems: 10) + #expect(loaded.status == .ready) + #expect(loaded.items.count == 1) + #expect(loaded.items.first?.text == "Hi") + } + + @Test func loaderReturnsEmptyWhenCachedEmpty() async { + await SessionPreviewCache.shared._testReset() + let snapshot = SessionMenuPreviewSnapshot(items: [], status: .empty) + await SessionPreviewCache.shared._testSet(snapshot: snapshot, for: "main") + + let loaded = await SessionMenuPreviewLoader.load(sessionKey: "main", maxItems: 10) + #expect(loaded.status == .empty) + #expect(loaded.items.isEmpty) + } +} diff --git a/apps/macos/Tests/OpenClawIPCTests/SettingsViewSmokeTests.swift b/apps/macos/Tests/OpenClawIPCTests/SettingsViewSmokeTests.swift new file mode 100644 index 0000000000000000000000000000000000000000..136091dbbe66ea56025932fea69ab27732c81252 --- /dev/null +++ b/apps/macos/Tests/OpenClawIPCTests/SettingsViewSmokeTests.swift @@ -0,0 +1,165 @@ +import SwiftUI +import Testing +@testable import OpenClaw + +@Suite(.serialized) +@MainActor +struct SettingsViewSmokeTests { + @Test func cronSettingsBuildsBody() { + let store = CronJobsStore(isPreview: true) + store.schedulerEnabled = false + store.schedulerStorePath = "/tmp/openclaw-cron-store.json" + + let job1 = CronJob( + id: "job-1", + agentId: "ops", + name: " Morning Check-in ", + description: nil, + enabled: true, + deleteAfterRun: nil, + createdAtMs: 1_700_000_000_000, + updatedAtMs: 1_700_000_100_000, + schedule: .cron(expr: "0 8 * * *", tz: "UTC"), + sessionTarget: .main, + wakeMode: .now, + payload: .systemEvent(text: "ping"), + isolation: nil, + state: CronJobState( + nextRunAtMs: 1_700_000_200_000, + runningAtMs: nil, + lastRunAtMs: 1_700_000_050_000, + lastStatus: "ok", + lastError: nil, + lastDurationMs: 123)) + + let job2 = CronJob( + id: "job-2", + agentId: nil, + name: "", + description: nil, + enabled: false, + deleteAfterRun: nil, + createdAtMs: 1_700_000_000_000, + updatedAtMs: 1_700_000_100_000, + schedule: .every(everyMs: 30000, anchorMs: nil), + sessionTarget: .isolated, + wakeMode: .nextHeartbeat, + payload: .agentTurn( + message: "hello", + thinking: "low", + timeoutSeconds: 30, + deliver: true, + channel: "sms", + to: "+15551234567", + bestEffortDeliver: true), + isolation: CronIsolation(postToMainPrefix: "[cron] "), + state: CronJobState( + nextRunAtMs: nil, + runningAtMs: nil, + lastRunAtMs: nil, + lastStatus: nil, + lastError: nil, + lastDurationMs: nil)) + + store.jobs = [job1, job2] + store.selectedJobId = job1.id + store.runEntries = [ + CronRunLogEntry( + ts: 1_700_000_050_000, + jobId: job1.id, + action: "finished", + status: "ok", + error: nil, + summary: "ok", + runAtMs: 1_700_000_050_000, + durationMs: 123, + nextRunAtMs: 1_700_000_200_000), + ] + + let view = CronSettings(store: store) + _ = view.body + } + + @Test func cronSettingsExercisesPrivateViews() { + CronSettings.exerciseForTesting() + } + + @Test func configSettingsBuildsBody() { + let view = ConfigSettings() + _ = view.body + } + + @Test func debugSettingsBuildsBody() { + let view = DebugSettings() + _ = view.body + } + + @Test func generalSettingsBuildsBody() { + let state = AppState(preview: true) + let view = GeneralSettings(state: state) + _ = view.body + } + + @Test func generalSettingsExercisesBranches() { + GeneralSettings.exerciseForTesting() + } + + @Test func sessionsSettingsBuildsBody() { + let view = SessionsSettings(rows: SessionRow.previewRows, isPreview: true) + _ = view.body + } + + @Test func instancesSettingsBuildsBody() { + let store = InstancesStore(isPreview: true) + store.instances = [ + InstanceInfo( + id: "local", + host: "this-mac", + ip: "127.0.0.1", + version: "1.0", + platform: "macos 15.0", + deviceFamily: "Mac", + modelIdentifier: "MacPreview", + lastInputSeconds: 12, + mode: "local", + reason: "test", + text: "test instance", + ts: Date().timeIntervalSince1970 * 1000), + ] + let view = InstancesSettings(store: store) + _ = view.body + } + + @Test func permissionsSettingsBuildsBody() { + let view = PermissionsSettings( + status: [ + .notifications: true, + .screenRecording: false, + ], + refresh: {}, + showOnboarding: {}) + _ = view.body + } + + @Test func settingsRootViewBuildsBody() { + let state = AppState(preview: true) + let view = SettingsRootView(state: state, updater: nil, initialTab: .general) + _ = view.body + } + + @Test func aboutSettingsBuildsBody() { + let view = AboutSettings(updater: nil) + _ = view.body + } + + @Test func voiceWakeSettingsBuildsBody() { + let state = AppState(preview: true) + let view = VoiceWakeSettings(state: state, isActive: false) + _ = view.body + } + + @Test func skillsSettingsBuildsBody() { + let view = SkillsSettings(state: .preview) + _ = view.body + } +} diff --git a/apps/macos/Tests/OpenClawIPCTests/SkillsSettingsSmokeTests.swift b/apps/macos/Tests/OpenClawIPCTests/SkillsSettingsSmokeTests.swift new file mode 100644 index 0000000000000000000000000000000000000000..560f3d2f50bf800a25d20026d6a63c7428549b0b --- /dev/null +++ b/apps/macos/Tests/OpenClawIPCTests/SkillsSettingsSmokeTests.swift @@ -0,0 +1,119 @@ +import OpenClawProtocol +import Testing +@testable import OpenClaw + +@Suite(.serialized) +@MainActor +struct SkillsSettingsSmokeTests { + @Test func skillsSettingsBuildsBodyWithSkillsRemote() { + let model = SkillsSettingsModel() + model.statusMessage = "Loaded" + model.skills = [ + SkillStatus( + name: "Needs Setup", + description: "Missing bins and env", + source: "openclaw-managed", + filePath: "/tmp/skills/needs-setup", + baseDir: "/tmp/skills", + skillKey: "needs-setup", + primaryEnv: "API_KEY", + emoji: "🧰", + homepage: "https://example.com/needs-setup", + always: false, + disabled: false, + eligible: false, + requirements: SkillRequirements( + bins: ["python3"], + env: ["API_KEY"], + config: ["skills.needs-setup"]), + missing: SkillMissing( + bins: ["python3"], + env: ["API_KEY"], + config: ["skills.needs-setup"]), + configChecks: [ + SkillStatusConfigCheck(path: "skills.needs-setup", value: AnyCodable(false), satisfied: false), + ], + install: [ + SkillInstallOption(id: "brew", kind: "brew", label: "brew install python", bins: ["python3"]), + ]), + SkillStatus( + name: "Ready Skill", + description: "All set", + source: "openclaw-bundled", + filePath: "/tmp/skills/ready", + baseDir: "/tmp/skills", + skillKey: "ready", + primaryEnv: nil, + emoji: "✅", + homepage: "https://example.com/ready", + always: false, + disabled: false, + eligible: true, + requirements: SkillRequirements(bins: [], env: [], config: []), + missing: SkillMissing(bins: [], env: [], config: []), + configChecks: [ + SkillStatusConfigCheck(path: "skills.ready", value: AnyCodable(true), satisfied: true), + SkillStatusConfigCheck(path: "skills.limit", value: AnyCodable(5), satisfied: true), + ], + install: []), + SkillStatus( + name: "Disabled Skill", + description: "Disabled in config", + source: "openclaw-extra", + filePath: "/tmp/skills/disabled", + baseDir: "/tmp/skills", + skillKey: "disabled", + primaryEnv: nil, + emoji: "🚫", + homepage: nil, + always: false, + disabled: true, + eligible: false, + requirements: SkillRequirements(bins: [], env: [], config: []), + missing: SkillMissing(bins: [], env: [], config: []), + configChecks: [], + install: []), + ] + + let state = AppState(preview: true) + state.connectionMode = .remote + var view = SkillsSettings(state: state, model: model) + view.setFilterForTesting("all") + _ = view.body + view.setFilterForTesting("needsSetup") + _ = view.body + } + + @Test func skillsSettingsBuildsBodyWithLocalMode() { + let model = SkillsSettingsModel() + model.skills = [ + SkillStatus( + name: "Local Skill", + description: "Local ready", + source: "openclaw-workspace", + filePath: "/tmp/skills/local", + baseDir: "/tmp/skills", + skillKey: "local", + primaryEnv: nil, + emoji: "🏠", + homepage: nil, + always: false, + disabled: false, + eligible: true, + requirements: SkillRequirements(bins: [], env: [], config: []), + missing: SkillMissing(bins: [], env: [], config: []), + configChecks: [], + install: []), + ] + + let state = AppState(preview: true) + state.connectionMode = .local + var view = SkillsSettings(state: state, model: model) + view.setFilterForTesting("ready") + _ = view.body + } + + @Test func skillsSettingsExercisesPrivateViews() { + SkillsSettings.exerciseForTesting() + } +} diff --git a/apps/macos/Tests/OpenClawIPCTests/TailscaleIntegrationSectionTests.swift b/apps/macos/Tests/OpenClawIPCTests/TailscaleIntegrationSectionTests.swift new file mode 100644 index 0000000000000000000000000000000000000000..fdfa96cbebbcac0294f9e77f8b056ead2fd75b47 --- /dev/null +++ b/apps/macos/Tests/OpenClawIPCTests/TailscaleIntegrationSectionTests.swift @@ -0,0 +1,48 @@ +import SwiftUI +import Testing +@testable import OpenClaw + +@Suite(.serialized) +@MainActor +struct TailscaleIntegrationSectionTests { + @Test func tailscaleSectionBuildsBodyWhenNotInstalled() { + let service = TailscaleService(isInstalled: false, isRunning: false, statusError: "not installed") + var view = TailscaleIntegrationSection(connectionMode: .local, isPaused: false) + view.setTestingService(service) + view.setTestingState(mode: "off", requireCredentials: false, statusMessage: "Idle") + _ = view.body + } + + @Test func tailscaleSectionBuildsBodyForServeMode() { + let service = TailscaleService( + isInstalled: true, + isRunning: true, + tailscaleHostname: "openclaw.tailnet.ts.net", + tailscaleIP: "100.64.0.1") + var view = TailscaleIntegrationSection(connectionMode: .local, isPaused: false) + view.setTestingService(service) + view.setTestingState( + mode: "serve", + requireCredentials: true, + password: "secret", + statusMessage: "Running") + _ = view.body + } + + @Test func tailscaleSectionBuildsBodyForFunnelMode() { + let service = TailscaleService( + isInstalled: true, + isRunning: false, + tailscaleHostname: nil, + tailscaleIP: nil, + statusError: "not running") + var view = TailscaleIntegrationSection(connectionMode: .remote, isPaused: false) + view.setTestingService(service) + view.setTestingState( + mode: "funnel", + requireCredentials: false, + statusMessage: "Needs start", + validationMessage: "Invalid token") + _ = view.body + } +} diff --git a/apps/macos/Tests/OpenClawIPCTests/TalkAudioPlayerTests.swift b/apps/macos/Tests/OpenClawIPCTests/TalkAudioPlayerTests.swift new file mode 100644 index 0000000000000000000000000000000000000000..bba233fa0c4b448e60b86df8d7cee71f54a5bbb3 --- /dev/null +++ b/apps/macos/Tests/OpenClawIPCTests/TalkAudioPlayerTests.swift @@ -0,0 +1,97 @@ +import Foundation +import Testing +@testable import OpenClaw + +@Suite(.serialized) struct TalkAudioPlayerTests { + @MainActor + @Test func playDoesNotHangWhenPlaybackEndsOrFails() async throws { + let wav = makeWav16Mono(sampleRate: 8000, samples: 80) + defer { _ = TalkAudioPlayer.shared.stop() } + + _ = try await withTimeout(seconds: 4.0) { + await TalkAudioPlayer.shared.play(data: wav) + } + + #expect(true) + } + + @MainActor + @Test func playDoesNotHangWhenPlayIsCalledTwice() async throws { + let wav = makeWav16Mono(sampleRate: 8000, samples: 800) + defer { _ = TalkAudioPlayer.shared.stop() } + + let first = Task { @MainActor in + await TalkAudioPlayer.shared.play(data: wav) + } + + await Task.yield() + _ = await TalkAudioPlayer.shared.play(data: wav) + + _ = try await withTimeout(seconds: 4.0) { + await first.value + } + #expect(true) + } +} + +private struct TimeoutError: Error {} + +private func withTimeout( + seconds: Double, + _ work: @escaping @Sendable () async throws -> T) async throws -> T +{ + try await withThrowingTaskGroup(of: T.self) { group in + group.addTask { + try await work() + } + group.addTask { + try await Task.sleep(nanoseconds: UInt64(seconds * 1_000_000_000)) + throw TimeoutError() + } + let result = try await group.next() + group.cancelAll() + guard let result else { throw TimeoutError() } + return result + } +} + +private func makeWav16Mono(sampleRate: UInt32, samples: Int) -> Data { + let channels: UInt16 = 1 + let bitsPerSample: UInt16 = 16 + let blockAlign = channels * (bitsPerSample / 8) + let byteRate = sampleRate * UInt32(blockAlign) + let dataSize = UInt32(samples) * UInt32(blockAlign) + + var data = Data() + data.append(contentsOf: [0x52, 0x49, 0x46, 0x46]) // RIFF + data.appendLEUInt32(36 + dataSize) + data.append(contentsOf: [0x57, 0x41, 0x56, 0x45]) // WAVE + + data.append(contentsOf: [0x66, 0x6D, 0x74, 0x20]) // fmt + data.appendLEUInt32(16) // PCM + data.appendLEUInt16(1) // audioFormat + data.appendLEUInt16(channels) + data.appendLEUInt32(sampleRate) + data.appendLEUInt32(byteRate) + data.appendLEUInt16(blockAlign) + data.appendLEUInt16(bitsPerSample) + + data.append(contentsOf: [0x64, 0x61, 0x74, 0x61]) // data + data.appendLEUInt32(dataSize) + + // Silence samples. + data.append(Data(repeating: 0, count: Int(dataSize))) + return data +} + +extension Data { + fileprivate mutating func appendLEUInt16(_ value: UInt16) { + var v = value.littleEndian + Swift.withUnsafeBytes(of: &v) { append(contentsOf: $0) } + } + + fileprivate mutating func appendLEUInt32(_ value: UInt32) { + var v = value.littleEndian + Swift.withUnsafeBytes(of: &v) { append(contentsOf: $0) } + } +} diff --git a/apps/macos/Tests/OpenClawIPCTests/TestIsolation.swift b/apps/macos/Tests/OpenClawIPCTests/TestIsolation.swift new file mode 100644 index 0000000000000000000000000000000000000000..1002b7ed30737af7c76d24e1719e1e0a900767f4 --- /dev/null +++ b/apps/macos/Tests/OpenClawIPCTests/TestIsolation.swift @@ -0,0 +1,116 @@ +import Foundation + +actor TestIsolationLock { + static let shared = TestIsolationLock() + + private var locked = false + private var waiters: [CheckedContinuation] = [] + + func acquire() async { + if !self.locked { + self.locked = true + return + } + await withCheckedContinuation { cont in + self.waiters.append(cont) + } + // `unlock()` resumed us; lock is now held for this caller. + } + + func release() { + if self.waiters.isEmpty { + self.locked = false + return + } + let next = self.waiters.removeFirst() + next.resume() + } +} + +@MainActor +enum TestIsolation { + static func withIsolatedState( + env: [String: String?] = [:], + defaults: [String: Any?] = [:], + _ body: () async throws -> T) async rethrows -> T + { + await TestIsolationLock.shared.acquire() + var previousEnv: [String: String?] = [:] + for (key, value) in env { + previousEnv[key] = getenv(key).map { String(cString: $0) } + if let value { + setenv(key, value, 1) + } else { + unsetenv(key) + } + } + + let userDefaults = UserDefaults.standard + var previousDefaults: [String: Any?] = [:] + for (key, value) in defaults { + previousDefaults[key] = userDefaults.object(forKey: key) + if let value { + userDefaults.set(value, forKey: key) + } else { + userDefaults.removeObject(forKey: key) + } + } + + do { + let result = try await body() + for (key, value) in previousDefaults { + if let value { + userDefaults.set(value, forKey: key) + } else { + userDefaults.removeObject(forKey: key) + } + } + for (key, value) in previousEnv { + if let value { + setenv(key, value, 1) + } else { + unsetenv(key) + } + } + await TestIsolationLock.shared.release() + return result + } catch { + for (key, value) in previousDefaults { + if let value { + userDefaults.set(value, forKey: key) + } else { + userDefaults.removeObject(forKey: key) + } + } + for (key, value) in previousEnv { + if let value { + setenv(key, value, 1) + } else { + unsetenv(key) + } + } + await TestIsolationLock.shared.release() + throw error + } + } + + static func withEnvValues( + _ values: [String: String?], + _ body: () async throws -> T) async rethrows -> T + { + try await self.withIsolatedState(env: values, defaults: [:], body) + } + + static func withUserDefaultsValues( + _ values: [String: Any?], + _ body: () async throws -> T) async rethrows -> T + { + try await self.withIsolatedState(env: [:], defaults: values, body) + } + + nonisolated static func tempConfigPath() -> String { + FileManager().temporaryDirectory + .appendingPathComponent("openclaw-test-config-\(UUID().uuidString).json") + .path + } +} diff --git a/apps/macos/Tests/OpenClawIPCTests/UtilitiesTests.swift b/apps/macos/Tests/OpenClawIPCTests/UtilitiesTests.swift new file mode 100644 index 0000000000000000000000000000000000000000..ddeef38dc1948c955cf4eaa623476187edb6de83 --- /dev/null +++ b/apps/macos/Tests/OpenClawIPCTests/UtilitiesTests.swift @@ -0,0 +1,83 @@ +import Foundation +import Testing +@testable import OpenClaw + +@Suite(.serialized) struct UtilitiesTests { + @Test func ageStringsCoverCommonWindows() { + let now = Date(timeIntervalSince1970: 1_000_000) + #expect(age(from: now, now: now) == "just now") + #expect(age(from: now.addingTimeInterval(-45), now: now) == "just now") + #expect(age(from: now.addingTimeInterval(-75), now: now) == "1 minute ago") + #expect(age(from: now.addingTimeInterval(-10 * 60), now: now) == "10m ago") + #expect(age(from: now.addingTimeInterval(-3600), now: now) == "1 hour ago") + #expect(age(from: now.addingTimeInterval(-5 * 3600), now: now) == "5h ago") + #expect(age(from: now.addingTimeInterval(-26 * 3600), now: now) == "yesterday") + #expect(age(from: now.addingTimeInterval(-3 * 86400), now: now) == "3d ago") + } + + @Test func parseSSHTargetSupportsUserPortAndDefaults() { + let parsed1 = CommandResolver.parseSSHTarget("alice@example.com:2222") + #expect(parsed1?.user == "alice") + #expect(parsed1?.host == "example.com") + #expect(parsed1?.port == 2222) + + let parsed2 = CommandResolver.parseSSHTarget("example.com") + #expect(parsed2?.user == nil) + #expect(parsed2?.host == "example.com") + #expect(parsed2?.port == 22) + + let parsed3 = CommandResolver.parseSSHTarget("bob@host") + #expect(parsed3?.user == "bob") + #expect(parsed3?.host == "host") + #expect(parsed3?.port == 22) + } + + @Test func sanitizedTargetStripsLeadingSSHPrefix() { + let defaults = UserDefaults(suiteName: "UtilitiesTests.\(UUID().uuidString)")! + defaults.set(AppState.ConnectionMode.remote.rawValue, forKey: connectionModeKey) + defaults.set("ssh alice@example.com", forKey: remoteTargetKey) + + let settings = CommandResolver.connectionSettings(defaults: defaults, configRoot: [:]) + #expect(settings.mode == .remote) + #expect(settings.target == "alice@example.com") + } + + @Test func gatewayEntrypointPrefersDistOverBin() throws { + let tmp = URL(fileURLWithPath: NSTemporaryDirectory(), isDirectory: true) + .appendingPathComponent(UUID().uuidString, isDirectory: true) + let dist = tmp.appendingPathComponent("dist/index.js") + let bin = tmp.appendingPathComponent("bin/openclaw.js") + try FileManager().createDirectory(at: dist.deletingLastPathComponent(), withIntermediateDirectories: true) + try FileManager().createDirectory(at: bin.deletingLastPathComponent(), withIntermediateDirectories: true) + FileManager().createFile(atPath: dist.path, contents: Data()) + FileManager().createFile(atPath: bin.path, contents: Data()) + + let entry = CommandResolver.gatewayEntrypoint(in: tmp) + #expect(entry == dist.path) + } + + @Test func logLocatorPicksNewestLogFile() throws { + let fm = FileManager() + let dir = URL(fileURLWithPath: "/tmp/openclaw", isDirectory: true) + try? fm.createDirectory(at: dir, withIntermediateDirectories: true) + + let older = dir.appendingPathComponent("openclaw-old-\(UUID().uuidString).log") + let newer = dir.appendingPathComponent("openclaw-new-\(UUID().uuidString).log") + fm.createFile(atPath: older.path, contents: Data("old".utf8)) + fm.createFile(atPath: newer.path, contents: Data("new".utf8)) + try fm.setAttributes([.modificationDate: Date(timeIntervalSinceNow: -100)], ofItemAtPath: older.path) + try fm.setAttributes([.modificationDate: Date()], ofItemAtPath: newer.path) + + let best = LogLocator.bestLogFile() + #expect(best?.lastPathComponent == newer.lastPathComponent) + + try? fm.removeItem(at: older) + try? fm.removeItem(at: newer) + } + + @Test func gatewayEntrypointNilWhenMissing() { + let tmp = URL(fileURLWithPath: NSTemporaryDirectory(), isDirectory: true) + .appendingPathComponent(UUID().uuidString, isDirectory: true) + #expect(CommandResolver.gatewayEntrypoint(in: tmp) == nil) + } +} diff --git a/apps/macos/Tests/OpenClawIPCTests/VoicePushToTalkHotkeyTests.swift b/apps/macos/Tests/OpenClawIPCTests/VoicePushToTalkHotkeyTests.swift new file mode 100644 index 0000000000000000000000000000000000000000..85cd72932fe514bcda13ceff443cb4ec2d9392e5 --- /dev/null +++ b/apps/macos/Tests/OpenClawIPCTests/VoicePushToTalkHotkeyTests.swift @@ -0,0 +1,37 @@ +import AppKit +import Testing +@testable import OpenClaw + +@Suite(.serialized) struct VoicePushToTalkHotkeyTests { + actor Counter { + private(set) var began = 0 + private(set) var ended = 0 + + func incBegin() { self.began += 1 } + func incEnd() { self.ended += 1 } + func snapshot() -> (began: Int, ended: Int) { (self.began, self.ended) } + } + + @Test func beginEndFiresOncePerHold() async { + let counter = Counter() + let hotkey = VoicePushToTalkHotkey( + beginAction: { await counter.incBegin() }, + endAction: { await counter.incEnd() }) + + await MainActor.run { + hotkey._testUpdateModifierState(keyCode: 61, modifierFlags: [.option]) + hotkey._testUpdateModifierState(keyCode: 61, modifierFlags: [.option]) + hotkey._testUpdateModifierState(keyCode: 61, modifierFlags: []) + } + + for _ in 0..<50 { + let snap = await counter.snapshot() + if snap.began == 1, snap.ended == 1 { break } + try? await Task.sleep(nanoseconds: 10_000_000) + } + + let snap = await counter.snapshot() + #expect(snap.began == 1) + #expect(snap.ended == 1) + } +} diff --git a/apps/macos/Tests/OpenClawIPCTests/VoicePushToTalkTests.swift b/apps/macos/Tests/OpenClawIPCTests/VoicePushToTalkTests.swift new file mode 100644 index 0000000000000000000000000000000000000000..4a69bfea941aeec5912fb0197c4d9a1a5b94e42c --- /dev/null +++ b/apps/macos/Tests/OpenClawIPCTests/VoicePushToTalkTests.swift @@ -0,0 +1,24 @@ +import Testing +@testable import OpenClaw + +@Suite struct VoicePushToTalkTests { + @Test func deltaTrimsCommittedPrefix() { + let delta = VoicePushToTalk._testDelta(committed: "hello ", current: "hello world again") + #expect(delta == "world again") + } + + @Test func deltaFallsBackWhenPrefixDiffers() { + let delta = VoicePushToTalk._testDelta(committed: "goodbye", current: "hello world") + #expect(delta == "hello world") + } + + @Test func attributedColorsDifferWhenNotFinal() { + let colors = VoicePushToTalk._testAttributedColors(isFinal: false) + #expect(colors.0 != colors.1) + } + + @Test func attributedColorsMatchWhenFinal() { + let colors = VoicePushToTalk._testAttributedColors(isFinal: true) + #expect(colors.0 == colors.1) + } +} diff --git a/apps/macos/Tests/OpenClawIPCTests/VoiceWakeForwarderTests.swift b/apps/macos/Tests/OpenClawIPCTests/VoiceWakeForwarderTests.swift new file mode 100644 index 0000000000000000000000000000000000000000..46971ac314c127b1f3af592638f92364c3c4c2be --- /dev/null +++ b/apps/macos/Tests/OpenClawIPCTests/VoiceWakeForwarderTests.swift @@ -0,0 +1,22 @@ +import Testing +@testable import OpenClaw + +@Suite(.serialized) struct VoiceWakeForwarderTests { + @Test func prefixedTranscriptUsesMachineName() { + let transcript = "hello world" + let prefixed = VoiceWakeForwarder.prefixedTranscript(transcript, machineName: "My-Mac") + + #expect(prefixed.starts(with: "User talked via voice recognition on")) + #expect(prefixed.contains("My-Mac")) + #expect(prefixed.hasSuffix("\n\nhello world")) + } + + @Test func forwardOptionsDefaults() { + let opts = VoiceWakeForwarder.ForwardOptions() + #expect(opts.sessionKey == "main") + #expect(opts.thinking == "low") + #expect(opts.deliver == true) + #expect(opts.to == nil) + #expect(opts.channel == .last) + } +} diff --git a/apps/macos/Tests/OpenClawIPCTests/VoiceWakeGlobalSettingsSyncTests.swift b/apps/macos/Tests/OpenClawIPCTests/VoiceWakeGlobalSettingsSyncTests.swift new file mode 100644 index 0000000000000000000000000000000000000000..9065f6b67c264b9d7926f748b6e327c6c4fd6e15 --- /dev/null +++ b/apps/macos/Tests/OpenClawIPCTests/VoiceWakeGlobalSettingsSyncTests.swift @@ -0,0 +1,56 @@ +import OpenClawProtocol +import Foundation +import Testing +@testable import OpenClaw + +@Suite(.serialized) struct VoiceWakeGlobalSettingsSyncTests { + @Test func appliesVoiceWakeChangedEventToAppState() async { + let previous = await MainActor.run { AppStateStore.shared.swabbleTriggerWords } + + await MainActor.run { + AppStateStore.shared.applyGlobalVoiceWakeTriggers(["before"]) + } + + let payload = OpenClawProtocol.AnyCodable(["triggers": ["openclaw", "computer"]]) + let evt = EventFrame( + type: "event", + event: "voicewake.changed", + payload: payload, + seq: nil, + stateversion: nil) + + await VoiceWakeGlobalSettingsSync.shared.handle(push: .event(evt)) + + let updated = await MainActor.run { AppStateStore.shared.swabbleTriggerWords } + #expect(updated == ["openclaw", "computer"]) + + await MainActor.run { + AppStateStore.shared.applyGlobalVoiceWakeTriggers(previous) + } + } + + @Test func ignoresVoiceWakeChangedEventWithInvalidPayload() async { + let previous = await MainActor.run { AppStateStore.shared.swabbleTriggerWords } + + await MainActor.run { + AppStateStore.shared.applyGlobalVoiceWakeTriggers(["before"]) + } + + let payload = OpenClawProtocol.AnyCodable(["unexpected": 123]) + let evt = EventFrame( + type: "event", + event: "voicewake.changed", + payload: payload, + seq: nil, + stateversion: nil) + + await VoiceWakeGlobalSettingsSync.shared.handle(push: .event(evt)) + + let updated = await MainActor.run { AppStateStore.shared.swabbleTriggerWords } + #expect(updated == ["before"]) + + await MainActor.run { + AppStateStore.shared.applyGlobalVoiceWakeTriggers(previous) + } + } +} diff --git a/apps/macos/Tests/OpenClawIPCTests/VoiceWakeHelpersTests.swift b/apps/macos/Tests/OpenClawIPCTests/VoiceWakeHelpersTests.swift new file mode 100644 index 0000000000000000000000000000000000000000..20ba7d7c4f5020d782b722cefadce97cc7ea9a4b --- /dev/null +++ b/apps/macos/Tests/OpenClawIPCTests/VoiceWakeHelpersTests.swift @@ -0,0 +1,35 @@ +import Testing +@testable import OpenClaw + +struct VoiceWakeHelpersTests { + @Test func sanitizeTriggersTrimsAndDropsEmpty() { + let cleaned = sanitizeVoiceWakeTriggers([" hi ", " ", "\n", "there"]) + #expect(cleaned == ["hi", "there"]) + } + + @Test func sanitizeTriggersFallsBackToDefaults() { + let cleaned = sanitizeVoiceWakeTriggers([" ", ""]) + #expect(cleaned == defaultVoiceWakeTriggers) + } + + @Test func sanitizeTriggersLimitsWordLength() { + let long = String(repeating: "x", count: voiceWakeMaxWordLength + 5) + let cleaned = sanitizeVoiceWakeTriggers(["ok", long]) + #expect(cleaned[1].count == voiceWakeMaxWordLength) + } + + @Test func sanitizeTriggersLimitsWordCount() { + let words = (1...voiceWakeMaxWords + 3).map { "w\($0)" } + let cleaned = sanitizeVoiceWakeTriggers(words) + #expect(cleaned.count == voiceWakeMaxWords) + } + + @Test func normalizeLocaleStripsCollation() { + #expect(normalizeLocaleIdentifier("en_US@collation=phonebook") == "en_US") + } + + @Test func normalizeLocaleStripsUnicodeExtensions() { + #expect(normalizeLocaleIdentifier("de-DE-u-co-phonebk") == "de-DE") + #expect(normalizeLocaleIdentifier("ja-JP-t-ja") == "ja-JP") + } +} diff --git a/apps/macos/Tests/OpenClawIPCTests/VoiceWakeOverlayControllerTests.swift b/apps/macos/Tests/OpenClawIPCTests/VoiceWakeOverlayControllerTests.swift new file mode 100644 index 0000000000000000000000000000000000000000..5e5636aee898b816556f0b306ac64ca597210b89 --- /dev/null +++ b/apps/macos/Tests/OpenClawIPCTests/VoiceWakeOverlayControllerTests.swift @@ -0,0 +1,68 @@ +import Foundation +import Testing +@testable import OpenClaw + +@Suite(.serialized) +@MainActor +struct VoiceWakeOverlayControllerTests { + @Test func overlayControllerLifecycleWithoutUI() async { + let controller = VoiceWakeOverlayController(enableUI: false) + let token = controller.startSession( + source: .wakeWord, + transcript: "hello", + attributed: nil, + forwardEnabled: true, + isFinal: false) + + #expect(controller.snapshot().token == token) + #expect(controller.snapshot().isVisible == true) + + controller.updatePartial(token: token, transcript: "hello world") + #expect(controller.snapshot().text == "hello world") + + controller.updateLevel(token: token, -0.5) + #expect(controller.model.level == 0) + try? await Task.sleep(nanoseconds: 120_000_000) + controller.updateLevel(token: token, 2.0) + #expect(controller.model.level == 1) + + controller.dismiss(token: token, reason: .explicit, outcome: .empty) + #expect(controller.snapshot().isVisible == false) + #expect(controller.snapshot().token == nil) + } + + @Test func evaluateTokenDropsMismatchAndNoActive() { + let active = UUID() + #expect(VoiceWakeOverlayController.evaluateToken(active: nil, incoming: active) == .dropNoActive) + #expect(VoiceWakeOverlayController.evaluateToken(active: active, incoming: UUID()) == .dropMismatch) + #expect(VoiceWakeOverlayController.evaluateToken(active: active, incoming: active) == .accept) + #expect(VoiceWakeOverlayController.evaluateToken(active: active, incoming: nil) == .accept) + } + + @Test func updateLevelThrottlesRapidChanges() async { + let controller = VoiceWakeOverlayController(enableUI: false) + let token = controller.startSession( + source: .wakeWord, + transcript: "level test", + attributed: nil, + forwardEnabled: false, + isFinal: false) + + controller.updateLevel(token: token, 0.25) + let first = controller.model.level + + controller.updateLevel(token: token, 0.9) + #expect(controller.model.level == first) + + controller.updateLevel(token: token, 0) + #expect(controller.model.level == 0) + + try? await Task.sleep(nanoseconds: 120_000_000) + controller.updateLevel(token: token, 0.9) + #expect(controller.model.level == 0.9) + } + + @Test func overlayControllerExercisesHelpers() async { + await VoiceWakeOverlayController.exerciseForTesting() + } +} diff --git a/apps/macos/Tests/OpenClawIPCTests/VoiceWakeOverlayTests.swift b/apps/macos/Tests/OpenClawIPCTests/VoiceWakeOverlayTests.swift new file mode 100644 index 0000000000000000000000000000000000000000..7e8b0a17f7053f9a31e7d4e58ef053dd16b21cab --- /dev/null +++ b/apps/macos/Tests/OpenClawIPCTests/VoiceWakeOverlayTests.swift @@ -0,0 +1,21 @@ +import Foundation +import Testing +@testable import OpenClaw + +@Suite struct VoiceWakeOverlayTests { + @Test func guardTokenDropsWhenNoActive() { + let outcome = VoiceWakeOverlayController.evaluateToken(active: nil, incoming: UUID()) + #expect(outcome == .dropNoActive) + } + + @Test func guardTokenAcceptsMatching() { + let token = UUID() + let outcome = VoiceWakeOverlayController.evaluateToken(active: token, incoming: token) + #expect(outcome == .accept) + } + + @Test func guardTokenDropsMismatchWithoutDismissing() { + let outcome = VoiceWakeOverlayController.evaluateToken(active: UUID(), incoming: UUID()) + #expect(outcome == .dropMismatch) + } +} diff --git a/apps/macos/Tests/OpenClawIPCTests/VoiceWakeOverlayViewSmokeTests.swift b/apps/macos/Tests/OpenClawIPCTests/VoiceWakeOverlayViewSmokeTests.swift new file mode 100644 index 0000000000000000000000000000000000000000..eaec98ab8b848c41ed4a60a343de2f897ae13a44 --- /dev/null +++ b/apps/macos/Tests/OpenClawIPCTests/VoiceWakeOverlayViewSmokeTests.swift @@ -0,0 +1,28 @@ +import SwiftUI +import Testing +@testable import OpenClaw + +@Suite(.serialized) +@MainActor +struct VoiceWakeOverlayViewSmokeTests { + @Test func overlayViewBuildsBodyInDisplayMode() { + let controller = VoiceWakeOverlayController(enableUI: false) + _ = controller.startSession(source: .wakeWord, transcript: "hello", forwardEnabled: true) + let view = VoiceWakeOverlayView(controller: controller) + _ = view.body + } + + @Test func overlayViewBuildsBodyInEditingMode() { + let controller = VoiceWakeOverlayController(enableUI: false) + let token = controller.startSession(source: .pushToTalk, transcript: "edit me", forwardEnabled: true) + controller.userBeganEditing() + controller.updateLevel(token: token, 0.6) + let view = VoiceWakeOverlayView(controller: controller) + _ = view.body + } + + @Test func closeButtonOverlayBuildsBody() { + let view = CloseButtonOverlay(isVisible: true, onHover: { _ in }, onClose: {}) + _ = view.body + } +} diff --git a/apps/macos/Tests/OpenClawIPCTests/VoiceWakeRuntimeTests.swift b/apps/macos/Tests/OpenClawIPCTests/VoiceWakeRuntimeTests.swift new file mode 100644 index 0000000000000000000000000000000000000000..3d92a32e0953bcc687ea3fcd46da4cab874953f1 --- /dev/null +++ b/apps/macos/Tests/OpenClawIPCTests/VoiceWakeRuntimeTests.swift @@ -0,0 +1,79 @@ +import Foundation +import SwabbleKit +import Testing +@testable import OpenClaw + +@Suite struct VoiceWakeRuntimeTests { + @Test func trimsAfterTriggerKeepsPostSpeech() { + let triggers = ["claude", "openclaw"] + let text = "hey Claude how are you" + #expect(VoiceWakeRuntime._testTrimmedAfterTrigger(text, triggers: triggers) == "how are you") + } + + @Test func trimsAfterTriggerReturnsOriginalWhenNoTrigger() { + let triggers = ["claude"] + let text = "good morning friend" + #expect(VoiceWakeRuntime._testTrimmedAfterTrigger(text, triggers: triggers) == text) + } + + @Test func trimsAfterFirstMatchingTrigger() { + let triggers = ["buddy", "claude"] + let text = "hello buddy this is after trigger claude also here" + #expect(VoiceWakeRuntime + ._testTrimmedAfterTrigger(text, triggers: triggers) == "this is after trigger claude also here") + } + + @Test func hasContentAfterTriggerFalseWhenOnlyTrigger() { + let triggers = ["openclaw"] + let text = "hey openclaw" + #expect(!VoiceWakeRuntime._testHasContentAfterTrigger(text, triggers: triggers)) + } + + @Test func hasContentAfterTriggerTrueWhenSpeechContinues() { + let triggers = ["claude"] + let text = "claude write a note" + #expect(VoiceWakeRuntime._testHasContentAfterTrigger(text, triggers: triggers)) + } + + @Test func gateRequiresGapBetweenTriggerAndCommand() { + let transcript = "hey openclaw do thing" + let segments = makeSegments( + transcript: transcript, + words: [ + ("hey", 0.0, 0.1), + ("openclaw", 0.2, 0.1), + ("do", 0.35, 0.1), + ("thing", 0.5, 0.1), + ]) + let config = WakeWordGateConfig(triggers: ["openclaw"], minPostTriggerGap: 0.3) + #expect(WakeWordGate.match(transcript: transcript, segments: segments, config: config) == nil) + } + + @Test func gateAcceptsGapAndExtractsCommand() { + let transcript = "hey openclaw do thing" + let segments = makeSegments( + transcript: transcript, + words: [ + ("hey", 0.0, 0.1), + ("openclaw", 0.2, 0.1), + ("do", 0.9, 0.1), + ("thing", 1.1, 0.1), + ]) + let config = WakeWordGateConfig(triggers: ["openclaw"], minPostTriggerGap: 0.3) + #expect(WakeWordGate.match(transcript: transcript, segments: segments, config: config)?.command == "do thing") + } +} + +private func makeSegments( + transcript: String, + words: [(String, TimeInterval, TimeInterval)]) +-> [WakeWordSegment] { + var searchStart = transcript.startIndex + var output: [WakeWordSegment] = [] + for (word, start, duration) in words { + let range = transcript.range(of: word, range: searchStart.. [WakeWordSegment] { + var searchStart = transcript.startIndex + var output: [WakeWordSegment] = [] + for (word, start, duration) in words { + let range = transcript.range(of: word, range: searchStart.. OpenClawChatHistoryPayload { + let json = """ + {"sessionKey":"\(sessionKey)","sessionId":null,"messages":[],"thinkingLevel":"off"} + """ + return try JSONDecoder().decode(OpenClawChatHistoryPayload.self, from: Data(json.utf8)) + } + + func sendMessage( + sessionKey _: String, + message _: String, + thinking _: String, + idempotencyKey _: String, + attachments _: [OpenClawChatAttachmentPayload]) async throws -> OpenClawChatSendResponse + { + let json = """ + {"runId":"\(UUID().uuidString)","status":"ok"} + """ + return try JSONDecoder().decode(OpenClawChatSendResponse.self, from: Data(json.utf8)) + } + + func requestHealth(timeoutMs _: Int) async throws -> Bool { true } + + func events() -> AsyncStream { + AsyncStream { continuation in + continuation.finish() + } + } + + func setActiveSessionKey(_: String) async throws {} + } + + @Test func windowControllerShowAndClose() { + let controller = WebChatSwiftUIWindowController( + sessionKey: "main", + presentation: .window, + transport: TestTransport()) + controller.show() + controller.close() + } + + @Test func panelControllerPresentAndClose() { + let anchor = { NSRect(x: 200, y: 400, width: 40, height: 40) } + let controller = WebChatSwiftUIWindowController( + sessionKey: "main", + presentation: .panel(anchorProvider: anchor), + transport: TestTransport()) + controller.presentAnchored(anchorProvider: anchor) + controller.close() + } +} diff --git a/apps/macos/Tests/OpenClawIPCTests/WideAreaGatewayDiscoveryTests.swift b/apps/macos/Tests/OpenClawIPCTests/WideAreaGatewayDiscoveryTests.swift new file mode 100644 index 0000000000000000000000000000000000000000..4bea51890ae42fe72090daf86bc3b30a62f28faa --- /dev/null +++ b/apps/macos/Tests/OpenClawIPCTests/WideAreaGatewayDiscoveryTests.swift @@ -0,0 +1,50 @@ +import Testing +@testable import OpenClawDiscovery + +@Suite +struct WideAreaGatewayDiscoveryTests { + @Test func discoversBeaconFromTailnetDnsSdFallback() { + setenv("OPENCLAW_WIDE_AREA_DOMAIN", "openclaw.internal", 1) + let statusJson = """ + { + "Self": { "TailscaleIPs": ["100.69.232.64"] }, + "Peer": { + "peer-1": { "TailscaleIPs": ["100.123.224.76"] } + } + } + """ + + let context = WideAreaGatewayDiscovery.DiscoveryContext( + tailscaleStatus: { statusJson }, + dig: { args, _ in + let recordType = args.last ?? "" + let nameserver = args.first(where: { $0.hasPrefix("@") }) ?? "" + if recordType == "PTR" { + if nameserver == "@100.123.224.76" { + return "steipetacstudio-gateway._openclaw-gw._tcp.openclaw.internal.\n" + } + return "" + } + if recordType == "SRV" { + return "0 0 18789 steipetacstudio.openclaw.internal." + } + if recordType == "TXT" { + return "\"displayName=Peter\\226\\128\\153s Mac Studio (OpenClaw)\" \"gatewayPort=18789\" \"tailnetDns=peters-mac-studio-1.sheep-coho.ts.net\" \"cliPath=/Users/steipete/openclaw/src/entry.ts\"" + } + return "" + }) + + let beacons = WideAreaGatewayDiscovery.discover( + timeoutSeconds: 2.0, + context: context) + + #expect(beacons.count == 1) + let beacon = beacons[0] + let expectedDisplay = "Peter\u{2019}s Mac Studio (OpenClaw)" + #expect(beacon.displayName == expectedDisplay) + #expect(beacon.port == 18789) + #expect(beacon.gatewayPort == 18789) + #expect(beacon.tailnetDns == "peters-mac-studio-1.sheep-coho.ts.net") + #expect(beacon.cliPath == "/Users/steipete/openclaw/src/entry.ts") + } +} diff --git a/apps/macos/Tests/OpenClawIPCTests/WindowPlacementTests.swift b/apps/macos/Tests/OpenClawIPCTests/WindowPlacementTests.swift new file mode 100644 index 0000000000000000000000000000000000000000..0afd3eb5b8819e40b68cd7815710770c1e987c3a --- /dev/null +++ b/apps/macos/Tests/OpenClawIPCTests/WindowPlacementTests.swift @@ -0,0 +1,85 @@ +import AppKit +import Testing +@testable import OpenClaw + +@Suite +@MainActor +struct WindowPlacementTests { + @Test + func centeredFrameZeroBoundsFallsBackToOrigin() { + let frame = WindowPlacement.centeredFrame(size: NSSize(width: 120, height: 80), in: NSRect.zero) + #expect(frame.origin == .zero) + #expect(frame.size == NSSize(width: 120, height: 80)) + } + + @Test + func centeredFrameClampsToBoundsAndCenters() { + let bounds = NSRect(x: 10, y: 20, width: 300, height: 200) + let frame = WindowPlacement.centeredFrame(size: NSSize(width: 600, height: 120), in: bounds) + #expect(frame.size.width == bounds.width) + #expect(frame.size.height == 120) + #expect(frame.minX == bounds.minX) + #expect(frame.midY == bounds.midY) + } + + @Test + func topRightFrameZeroBoundsFallsBackToOrigin() { + let frame = WindowPlacement.topRightFrame( + size: NSSize(width: 120, height: 80), + padding: 12, + in: NSRect.zero) + #expect(frame.origin == .zero) + #expect(frame.size == NSSize(width: 120, height: 80)) + } + + @Test + func topRightFrameClampsToBoundsAndAppliesPadding() { + let bounds = NSRect(x: 10, y: 20, width: 300, height: 200) + let frame = WindowPlacement.topRightFrame( + size: NSSize(width: 400, height: 50), + padding: 8, + in: bounds) + #expect(frame.size.width == bounds.width) + #expect(frame.size.height == 50) + #expect(frame.maxX == bounds.maxX - 8) + #expect(frame.maxY == bounds.maxY - 8) + } + + @Test + func ensureOnScreenUsesFallbackWhenWindowOffscreen() { + let window = NSWindow( + contentRect: NSRect(x: 100_000, y: 100_000, width: 200, height: 120), + styleMask: [.borderless], + backing: .buffered, + defer: false) + + WindowPlacement.ensureOnScreen( + window: window, + defaultSize: NSSize(width: 200, height: 120), + fallback: { _ in NSRect(x: 11, y: 22, width: 33, height: 44) }) + + #expect(window.frame == NSRect(x: 11, y: 22, width: 33, height: 44)) + } + + @Test + func ensureOnScreenDoesNotMoveVisibleWindow() { + let screen = NSScreen.main ?? NSScreen.screens.first + #expect(screen != nil) + guard let screen else { return } + + let visible = screen.visibleFrame.insetBy(dx: 40, dy: 40) + let window = NSWindow( + contentRect: NSRect(x: visible.minX, y: visible.minY, width: 200, height: 120), + styleMask: [.titled], + backing: .buffered, + defer: false) + let original = window.frame + + WindowPlacement.ensureOnScreen( + window: window, + defaultSize: NSSize(width: 200, height: 120), + fallback: { _ in NSRect(x: 11, y: 22, width: 33, height: 44) }) + + #expect(window.frame == original) + } +} diff --git a/apps/macos/Tests/OpenClawIPCTests/WorkActivityStoreTests.swift b/apps/macos/Tests/OpenClawIPCTests/WorkActivityStoreTests.swift new file mode 100644 index 0000000000000000000000000000000000000000..7882706430ddb89cca8f66bd27bccd78f4954519 --- /dev/null +++ b/apps/macos/Tests/OpenClawIPCTests/WorkActivityStoreTests.swift @@ -0,0 +1,99 @@ +import OpenClawProtocol +import Foundation +import Testing +@testable import OpenClaw + +@Suite +@MainActor +struct WorkActivityStoreTests { + @Test func mainSessionJobPreemptsOther() { + let store = WorkActivityStore() + + store.handleJob(sessionKey: "discord:group:1", state: "started") + #expect(store.iconState == .workingOther(.job)) + #expect(store.current?.sessionKey == "discord:group:1") + + store.handleJob(sessionKey: "main", state: "started") + #expect(store.iconState == .workingMain(.job)) + #expect(store.current?.sessionKey == "main") + + store.handleJob(sessionKey: "main", state: "finished") + #expect(store.iconState == .workingOther(.job)) + #expect(store.current?.sessionKey == "discord:group:1") + + store.handleJob(sessionKey: "discord:group:1", state: "finished") + #expect(store.iconState == .idle) + #expect(store.current == nil) + } + + @Test func jobStaysWorkingAfterToolResultGrace() async { + let store = WorkActivityStore() + + store.handleJob(sessionKey: "main", state: "started") + #expect(store.iconState == .workingMain(.job)) + + store.handleTool( + sessionKey: "main", + phase: "start", + name: "read", + meta: nil, + args: ["path": AnyCodable("/tmp/file.txt")]) + #expect(store.iconState == .workingMain(.tool(.read))) + + store.handleTool( + sessionKey: "main", + phase: "result", + name: "read", + meta: nil, + args: ["path": AnyCodable("/tmp/file.txt")]) + + for _ in 0..<50 { + if store.iconState == .workingMain(.job) { break } + try? await Task.sleep(nanoseconds: 100_000_000) + } + #expect(store.iconState == .workingMain(.job)) + + store.handleJob(sessionKey: "main", state: "done") + #expect(store.iconState == .idle) + } + + @Test func toolLabelExtractsFirstLineAndShortensHome() { + let store = WorkActivityStore() + let home = NSHomeDirectory() + + store.handleTool( + sessionKey: "main", + phase: "start", + name: "bash", + meta: nil, + args: [ + "command": AnyCodable("echo hi\necho bye"), + "path": AnyCodable("\(home)/Projects/openclaw"), + ]) + + #expect(store.current?.label == "bash: echo hi") + #expect(store.iconState == .workingMain(.tool(.bash))) + + store.handleTool( + sessionKey: "main", + phase: "start", + name: "read", + meta: nil, + args: ["path": AnyCodable("\(home)/secret.txt")]) + + #expect(store.current?.label == "read: ~/secret.txt") + #expect(store.iconState == .workingMain(.tool(.read))) + } + + @Test func resolveIconStateHonorsOverrideSelection() { + let store = WorkActivityStore() + store.handleJob(sessionKey: "main", state: "started") + #expect(store.iconState == .workingMain(.job)) + + store.resolveIconState(override: .idle) + #expect(store.iconState == .idle) + + store.resolveIconState(override: .otherEdit) + #expect(store.iconState == .overridden(.tool(.edit))) + } +} diff --git a/apps/shared/OpenClawKit/Package.swift b/apps/shared/OpenClawKit/Package.swift new file mode 100644 index 0000000000000000000000000000000000000000..5c8132d2c9bf5f7e2e3ffa471ee78cfa88d15c74 --- /dev/null +++ b/apps/shared/OpenClawKit/Package.swift @@ -0,0 +1,61 @@ +// swift-tools-version: 6.2 + +import PackageDescription + +let package = Package( + name: "OpenClawKit", + platforms: [ + .iOS(.v18), + .macOS(.v15), + ], + products: [ + .library(name: "OpenClawProtocol", targets: ["OpenClawProtocol"]), + .library(name: "OpenClawKit", targets: ["OpenClawKit"]), + .library(name: "OpenClawChatUI", targets: ["OpenClawChatUI"]), + ], + dependencies: [ + .package(url: "https://github.com/steipete/ElevenLabsKit", exact: "0.1.0"), + .package(url: "https://github.com/gonzalezreal/textual", exact: "0.3.1"), + ], + targets: [ + .target( + name: "OpenClawProtocol", + path: "Sources/OpenClawProtocol", + swiftSettings: [ + .enableUpcomingFeature("StrictConcurrency"), + ]), + .target( + name: "OpenClawKit", + dependencies: [ + "OpenClawProtocol", + .product(name: "ElevenLabsKit", package: "ElevenLabsKit"), + ], + path: "Sources/OpenClawKit", + resources: [ + .process("Resources"), + ], + swiftSettings: [ + .enableUpcomingFeature("StrictConcurrency"), + ]), + .target( + name: "OpenClawChatUI", + dependencies: [ + "OpenClawKit", + .product( + name: "Textual", + package: "textual", + condition: .when(platforms: [.macOS, .iOS])), + ], + path: "Sources/OpenClawChatUI", + swiftSettings: [ + .enableUpcomingFeature("StrictConcurrency"), + ]), + .testTarget( + name: "OpenClawKitTests", + dependencies: ["OpenClawKit", "OpenClawChatUI"], + path: "Tests/OpenClawKitTests", + swiftSettings: [ + .enableUpcomingFeature("StrictConcurrency"), + .enableExperimentalFeature("SwiftTesting"), + ]), + ]) diff --git a/apps/shared/OpenClawKit/Sources/OpenClawChatUI/AssistantTextParser.swift b/apps/shared/OpenClawKit/Sources/OpenClawChatUI/AssistantTextParser.swift new file mode 100644 index 0000000000000000000000000000000000000000..c4395adfaea824eeaad05fd00ee1cdfdeab77d75 --- /dev/null +++ b/apps/shared/OpenClawKit/Sources/OpenClawChatUI/AssistantTextParser.swift @@ -0,0 +1,139 @@ +import Foundation + +struct AssistantTextSegment: Identifiable { + enum Kind { + case thinking + case response + } + + let id = UUID() + let kind: Kind + let text: String +} + +enum AssistantTextParser { + static func segments(from raw: String) -> [AssistantTextSegment] { + let trimmed = raw.trimmingCharacters(in: .whitespacesAndNewlines) + guard !trimmed.isEmpty else { return [] } + guard raw.contains("<") else { + return [AssistantTextSegment(kind: .response, text: trimmed)] + } + + var segments: [AssistantTextSegment] = [] + var cursor = raw.startIndex + var currentKind: AssistantTextSegment.Kind = .response + var matchedTag = false + + while let match = self.nextTag(in: raw, from: cursor) { + matchedTag = true + if match.range.lowerBound > cursor { + self.appendSegment(kind: currentKind, text: raw[cursor..", range: match.range.upperBound.. Bool { + !self.segments(from: raw).isEmpty + } + + private enum TagKind { + case think + case final + } + + private struct TagMatch { + let kind: TagKind + let closing: Bool + let range: Range + } + + private static func nextTag(in text: String, from start: String.Index) -> TagMatch? { + let candidates: [TagMatch] = [ + self.findTagStart(tag: "think", closing: false, in: text, from: start).map { + TagMatch(kind: .think, closing: false, range: $0) + }, + self.findTagStart(tag: "think", closing: true, in: text, from: start).map { + TagMatch(kind: .think, closing: true, range: $0) + }, + self.findTagStart(tag: "final", closing: false, in: text, from: start).map { + TagMatch(kind: .final, closing: false, range: $0) + }, + self.findTagStart(tag: "final", closing: true, in: text, from: start).map { + TagMatch(kind: .final, closing: true, range: $0) + }, + ].compactMap(\.self) + + return candidates.min { $0.range.lowerBound < $1.range.lowerBound } + } + + private static func findTagStart( + tag: String, + closing: Bool, + in text: String, + from start: String.Index) -> Range? + { + let token = closing ? "" || boundary.isWhitespace || (!closing && boundary == "/") + if isBoundary { + return range + } + searchRange = boundaryIndex..) -> Bool { + var cursor = tagEnd.lowerBound + while cursor > text.startIndex { + cursor = text.index(before: cursor) + let char = text[cursor] + if char.isWhitespace { continue } + return char == "/" + } + return false + } + + private static func appendSegment( + kind: AssistantTextSegment.Kind, + text: Substring, + to segments: inout [AssistantTextSegment]) + { + let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines) + guard !trimmed.isEmpty else { return } + segments.append(AssistantTextSegment(kind: kind, text: trimmed)) + } +} diff --git a/apps/shared/OpenClawKit/Sources/OpenClawChatUI/ChatComposer.swift b/apps/shared/OpenClawKit/Sources/OpenClawChatUI/ChatComposer.swift new file mode 100644 index 0000000000000000000000000000000000000000..95a5ac3e5842f6ec6603f866c1cfc533ae3b8c1c --- /dev/null +++ b/apps/shared/OpenClawKit/Sources/OpenClawChatUI/ChatComposer.swift @@ -0,0 +1,489 @@ +import Foundation +import Observation +import SwiftUI + +#if !os(macOS) +import PhotosUI +import UniformTypeIdentifiers +#endif + +@MainActor +struct OpenClawChatComposer: View { + @Bindable var viewModel: OpenClawChatViewModel + let style: OpenClawChatView.Style + let showsSessionSwitcher: Bool + + #if !os(macOS) + @State private var pickerItems: [PhotosPickerItem] = [] + @FocusState private var isFocused: Bool + #else + @State private var shouldFocusTextView = false + #endif + + var body: some View { + VStack(alignment: .leading, spacing: 4) { + if self.showsToolbar { + HStack(spacing: 6) { + if self.showsSessionSwitcher { + self.sessionPicker + } + self.thinkingPicker + Spacer() + self.refreshButton + self.attachmentPicker + } + } + + if self.showsAttachments, !self.viewModel.attachments.isEmpty { + self.attachmentsStrip + } + + self.editor + } + .padding(self.composerPadding) + .background { + let cornerRadius: CGFloat = 18 + + #if os(macOS) + if self.style == .standard { + let shape = UnevenRoundedRectangle( + cornerRadii: RectangleCornerRadii( + topLeading: 0, + bottomLeading: cornerRadius, + bottomTrailing: cornerRadius, + topTrailing: 0), + style: .continuous) + shape + .fill(OpenClawChatTheme.composerBackground) + .overlay(shape.strokeBorder(OpenClawChatTheme.composerBorder, lineWidth: 1)) + .shadow(color: .black.opacity(0.12), radius: 12, y: 6) + } else { + let shape = RoundedRectangle(cornerRadius: cornerRadius, style: .continuous) + shape + .fill(OpenClawChatTheme.composerBackground) + .overlay(shape.strokeBorder(OpenClawChatTheme.composerBorder, lineWidth: 1)) + .shadow(color: .black.opacity(0.12), radius: 12, y: 6) + } + #else + let shape = RoundedRectangle(cornerRadius: cornerRadius, style: .continuous) + shape + .fill(OpenClawChatTheme.composerBackground) + .overlay(shape.strokeBorder(OpenClawChatTheme.composerBorder, lineWidth: 1)) + .shadow(color: .black.opacity(0.12), radius: 12, y: 6) + #endif + } + #if os(macOS) + .onDrop(of: [.fileURL], isTargeted: nil) { providers in + self.handleDrop(providers) + } + .onAppear { + self.shouldFocusTextView = true + } + #endif + } + + private var thinkingPicker: some View { + Picker("Thinking", selection: self.$viewModel.thinkingLevel) { + Text("Off").tag("off") + Text("Low").tag("low") + Text("Medium").tag("medium") + Text("High").tag("high") + } + .labelsHidden() + .pickerStyle(.menu) + .controlSize(.small) + .frame(maxWidth: 140, alignment: .leading) + } + + private var sessionPicker: some View { + Picker( + "Session", + selection: Binding( + get: { self.viewModel.sessionKey }, + set: { next in self.viewModel.switchSession(to: next) })) + { + ForEach(self.viewModel.sessionChoices, id: \.key) { session in + Text(session.displayName ?? session.key) + .font(.system(.caption, design: .monospaced)) + .tag(session.key) + } + } + .labelsHidden() + .pickerStyle(.menu) + .controlSize(.small) + .frame(maxWidth: 160, alignment: .leading) + .help("Session") + } + + @ViewBuilder + private var attachmentPicker: some View { + #if os(macOS) + Button { + self.pickFilesMac() + } label: { + Image(systemName: "paperclip") + } + .help("Add Image") + .buttonStyle(.bordered) + .controlSize(.small) + #else + PhotosPicker(selection: self.$pickerItems, maxSelectionCount: 8, matching: .images) { + Image(systemName: "paperclip") + } + .help("Add Image") + .buttonStyle(.bordered) + .controlSize(.small) + .onChange(of: self.pickerItems) { _, newItems in + Task { await self.loadPhotosPickerItems(newItems) } + } + #endif + } + + private var attachmentsStrip: some View { + ScrollView(.horizontal, showsIndicators: false) { + HStack(spacing: 6) { + ForEach( + self.viewModel.attachments, + id: \OpenClawPendingAttachment.id) + { (att: OpenClawPendingAttachment) in + HStack(spacing: 6) { + if let img = att.preview { + OpenClawPlatformImageFactory.image(img) + .resizable() + .scaledToFill() + .frame(width: 22, height: 22) + .clipShape(RoundedRectangle(cornerRadius: 6, style: .continuous)) + } else { + Image(systemName: "photo") + } + + Text(att.fileName) + .lineLimit(1) + + Button { + self.viewModel.removeAttachment(att.id) + } label: { + Image(systemName: "xmark.circle.fill") + } + .buttonStyle(.plain) + } + .padding(.horizontal, 8) + .padding(.vertical, 5) + .background(Color.accentColor.opacity(0.08)) + .clipShape(Capsule()) + } + } + } + } + + private var editor: some View { + VStack(alignment: .leading, spacing: 8) { + self.editorOverlay + + Rectangle() + .fill(OpenClawChatTheme.divider) + .frame(height: 1) + .padding(.horizontal, 2) + + HStack(alignment: .center, spacing: 8) { + if self.showsConnectionPill { + self.connectionPill + } + Spacer(minLength: 0) + self.sendButton + } + } + .padding(.horizontal, 10) + .padding(.vertical, 8) + .background( + RoundedRectangle(cornerRadius: 12, style: .continuous) + .fill(OpenClawChatTheme.composerField) + .overlay( + RoundedRectangle(cornerRadius: 12, style: .continuous) + .strokeBorder(OpenClawChatTheme.composerBorder))) + .padding(self.editorPadding) + } + + private var connectionPill: some View { + HStack(spacing: 6) { + Circle() + .fill(self.viewModel.healthOK ? .green : .orange) + .frame(width: 7, height: 7) + Text(self.activeSessionLabel) + .font(.caption2.weight(.semibold)) + Text(self.viewModel.healthOK ? "Connected" : "Connecting…") + .font(.caption2) + .foregroundStyle(.secondary) + } + .padding(.horizontal, 8) + .padding(.vertical, 4) + .background(OpenClawChatTheme.subtleCard) + .clipShape(Capsule()) + } + + private var activeSessionLabel: String { + let match = self.viewModel.sessions.first { $0.key == self.viewModel.sessionKey } + let trimmed = match?.displayName?.trimmingCharacters(in: .whitespacesAndNewlines) ?? "" + return trimmed.isEmpty ? self.viewModel.sessionKey : trimmed + } + + private var editorOverlay: some View { + ZStack(alignment: .topLeading) { + if self.viewModel.input.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty { + Text("Message OpenClaw…") + .foregroundStyle(.tertiary) + .padding(.horizontal, 4) + .padding(.vertical, 4) + } + + #if os(macOS) + ChatComposerTextView(text: self.$viewModel.input, shouldFocus: self.$shouldFocusTextView) { + self.viewModel.send() + } + .frame(minHeight: self.textMinHeight, idealHeight: self.textMinHeight, maxHeight: self.textMaxHeight) + .padding(.horizontal, 4) + .padding(.vertical, 3) + #else + TextEditor(text: self.$viewModel.input) + .font(.system(size: 15)) + .scrollContentBackground(.hidden) + .frame( + minHeight: self.textMinHeight, + idealHeight: self.textMinHeight, + maxHeight: self.textMaxHeight) + .padding(.horizontal, 4) + .padding(.vertical, 4) + .focused(self.$isFocused) + #endif + } + } + + private var sendButton: some View { + Group { + if self.viewModel.pendingRunCount > 0 { + Button { + self.viewModel.abort() + } label: { + if self.viewModel.isAborting { + ProgressView().controlSize(.mini) + } else { + Image(systemName: "stop.fill") + .font(.system(size: 13, weight: .semibold)) + } + } + .buttonStyle(.plain) + .foregroundStyle(.white) + .padding(6) + .background(Circle().fill(Color.red)) + .disabled(self.viewModel.isAborting) + } else { + Button { + self.viewModel.send() + } label: { + if self.viewModel.isSending { + ProgressView().controlSize(.mini) + } else { + Image(systemName: "arrow.up") + .font(.system(size: 13, weight: .semibold)) + } + } + .buttonStyle(.plain) + .foregroundStyle(.white) + .padding(6) + .background(Circle().fill(Color.accentColor)) + .disabled(!self.viewModel.canSend) + } + } + } + + private var refreshButton: some View { + Button { + self.viewModel.refresh() + } label: { + Image(systemName: "arrow.clockwise") + } + .buttonStyle(.bordered) + .controlSize(.small) + .help("Refresh") + } + + private var showsToolbar: Bool { + self.style == .standard + } + + private var showsAttachments: Bool { + self.style == .standard + } + + private var showsConnectionPill: Bool { + self.style == .standard + } + + private var composerPadding: CGFloat { + self.style == .onboarding ? 5 : 6 + } + + private var editorPadding: CGFloat { + self.style == .onboarding ? 5 : 6 + } + + private var textMinHeight: CGFloat { + self.style == .onboarding ? 24 : 28 + } + + private var textMaxHeight: CGFloat { + self.style == .onboarding ? 52 : 64 + } + + #if os(macOS) + private func pickFilesMac() { + let panel = NSOpenPanel() + panel.title = "Select image attachments" + panel.allowsMultipleSelection = true + panel.canChooseDirectories = false + panel.allowedContentTypes = [.image] + panel.begin { resp in + guard resp == .OK else { return } + self.viewModel.addAttachments(urls: panel.urls) + } + } + + private func handleDrop(_ providers: [NSItemProvider]) -> Bool { + let fileProviders = providers.filter { $0.hasItemConformingToTypeIdentifier(UTType.fileURL.identifier) } + guard !fileProviders.isEmpty else { return false } + for item in fileProviders { + item.loadItem(forTypeIdentifier: UTType.fileURL.identifier, options: nil) { item, _ in + guard let data = item as? Data, + let url = URL(dataRepresentation: data, relativeTo: nil) + else { return } + Task { @MainActor in + self.viewModel.addAttachments(urls: [url]) + } + } + } + return true + } + #else + private func loadPhotosPickerItems(_ items: [PhotosPickerItem]) async { + for item in items { + do { + guard let data = try await item.loadTransferable(type: Data.self) else { continue } + let type = item.supportedContentTypes.first ?? .image + let ext = type.preferredFilenameExtension ?? "jpg" + let mime = type.preferredMIMEType ?? "image/jpeg" + let name = "photo-\(UUID().uuidString.prefix(8)).\(ext)" + self.viewModel.addImageAttachment(data: data, fileName: name, mimeType: mime) + } catch { + self.viewModel.errorText = error.localizedDescription + } + } + self.pickerItems = [] + } + #endif +} + +#if os(macOS) +import AppKit +import UniformTypeIdentifiers + +private struct ChatComposerTextView: NSViewRepresentable { + @Binding var text: String + @Binding var shouldFocus: Bool + var onSend: () -> Void + + func makeCoordinator() -> Coordinator { Coordinator(self) } + + func makeNSView(context: Context) -> NSScrollView { + let textView = ChatComposerNSTextView() + textView.delegate = context.coordinator + textView.drawsBackground = false + textView.isRichText = false + textView.isAutomaticQuoteSubstitutionEnabled = false + textView.isAutomaticTextReplacementEnabled = false + textView.isAutomaticDashSubstitutionEnabled = false + textView.isAutomaticSpellingCorrectionEnabled = false + textView.font = .systemFont(ofSize: 14, weight: .regular) + textView.textContainer?.lineBreakMode = .byWordWrapping + textView.textContainer?.lineFragmentPadding = 0 + textView.textContainerInset = NSSize(width: 2, height: 4) + textView.focusRingType = .none + + textView.minSize = .zero + textView.maxSize = NSSize(width: CGFloat.greatestFiniteMagnitude, height: CGFloat.greatestFiniteMagnitude) + textView.isHorizontallyResizable = false + textView.isVerticallyResizable = true + textView.autoresizingMask = [.width] + textView.textContainer?.containerSize = NSSize(width: 0, height: CGFloat.greatestFiniteMagnitude) + textView.textContainer?.widthTracksTextView = true + + textView.string = self.text + textView.onSend = { [weak textView] in + textView?.window?.makeFirstResponder(nil) + self.onSend() + } + + let scroll = NSScrollView() + scroll.drawsBackground = false + scroll.borderType = .noBorder + scroll.hasVerticalScroller = true + scroll.autohidesScrollers = true + scroll.scrollerStyle = .overlay + scroll.hasHorizontalScroller = false + scroll.documentView = textView + return scroll + } + + func updateNSView(_ scrollView: NSScrollView, context: Context) { + guard let textView = scrollView.documentView as? ChatComposerNSTextView else { return } + + if self.shouldFocus, let window = scrollView.window { + window.makeFirstResponder(textView) + self.shouldFocus = false + } + + let isEditing = scrollView.window?.firstResponder == textView + + // Always allow clearing the text (e.g. after send), even while editing. + // Only skip other updates while editing to avoid cursor jumps. + let shouldClear = self.text.isEmpty && !textView.string.isEmpty + if isEditing, !shouldClear { return } + + if textView.string != self.text { + context.coordinator.isProgrammaticUpdate = true + defer { context.coordinator.isProgrammaticUpdate = false } + textView.string = self.text + } + } + + final class Coordinator: NSObject, NSTextViewDelegate { + var parent: ChatComposerTextView + var isProgrammaticUpdate = false + + init(_ parent: ChatComposerTextView) { self.parent = parent } + + func textDidChange(_ notification: Notification) { + guard !self.isProgrammaticUpdate else { return } + guard let view = notification.object as? NSTextView else { return } + guard view.window?.firstResponder === view else { return } + self.parent.text = view.string + } + } +} + +private final class ChatComposerNSTextView: NSTextView { + var onSend: (() -> Void)? + + override func keyDown(with event: NSEvent) { + let isReturn = event.keyCode == 36 + if isReturn { + if event.modifierFlags.contains(.shift) { + super.insertNewline(nil) + return + } + self.onSend?() + return + } + super.keyDown(with: event) + } +} +#endif diff --git a/apps/shared/OpenClawKit/Sources/OpenClawChatUI/ChatMarkdownPreprocessor.swift b/apps/shared/OpenClawKit/Sources/OpenClawChatUI/ChatMarkdownPreprocessor.swift new file mode 100644 index 0000000000000000000000000000000000000000..f435eab2dca66c11812b1abd04492afccde14af3 --- /dev/null +++ b/apps/shared/OpenClawKit/Sources/OpenClawChatUI/ChatMarkdownPreprocessor.swift @@ -0,0 +1,51 @@ +import Foundation + +enum ChatMarkdownPreprocessor { + struct InlineImage: Identifiable { + let id = UUID() + let label: String + let image: OpenClawPlatformImage? + } + + struct Result { + let cleaned: String + let images: [InlineImage] + } + + static func preprocess(markdown raw: String) -> Result { + let pattern = #"!\[([^\]]*)\]\((data:image\/[^;]+;base64,[^)]+)\)"# + guard let re = try? NSRegularExpression(pattern: pattern) else { + return Result(cleaned: raw, images: []) + } + + let ns = raw as NSString + let matches = re.matches(in: raw, range: NSRange(location: 0, length: ns.length)) + if matches.isEmpty { return Result(cleaned: raw, images: []) } + + var images: [InlineImage] = [] + var cleaned = raw + + for match in matches.reversed() { + guard match.numberOfRanges >= 3 else { continue } + let label = ns.substring(with: match.range(at: 1)) + let dataURL = ns.substring(with: match.range(at: 2)) + + let image: OpenClawPlatformImage? = { + guard let comma = dataURL.firstIndex(of: ",") else { return nil } + let b64 = String(dataURL[dataURL.index(after: comma)...]) + guard let data = Data(base64Encoded: b64) else { return nil } + return OpenClawPlatformImage(data: data) + }() + images.append(InlineImage(label: label, image: image)) + + let start = cleaned.index(cleaned.startIndex, offsetBy: match.range.location) + let end = cleaned.index(start, offsetBy: match.range.length) + cleaned.replaceSubrange(start.. some View { + Group { + if self.variant == .compact { + content.textual.structuredTextStyle(.default) + } else { + content.textual.structuredTextStyle(.gitHub) + } + } + .font(self.font) + .foregroundStyle(self.textColor) + .textual.inlineStyle(self.inlineStyle) + .textual.textSelection(.enabled) + } + + private var inlineStyle: InlineStyle { + let linkColor: Color = self.context == .user ? self.textColor : .accentColor + let codeScale: CGFloat = self.variant == .compact ? 0.85 : 0.9 + return InlineStyle() + .code(.monospaced, .fontScale(codeScale)) + .link(.foregroundColor(linkColor)) + } +} + +@MainActor +private struct InlineImageList: View { + let images: [ChatMarkdownPreprocessor.InlineImage] + + var body: some View { + ForEach(images, id: \.id) { item in + if let img = item.image { + OpenClawPlatformImageFactory.image(img) + .resizable() + .scaledToFit() + .frame(maxHeight: 260) + .clipShape(RoundedRectangle(cornerRadius: 12, style: .continuous)) + .overlay( + RoundedRectangle(cornerRadius: 12, style: .continuous) + .strokeBorder(Color.white.opacity(0.12), lineWidth: 1)) + } else { + Text(item.label.isEmpty ? "Image" : item.label) + .font(.footnote) + .foregroundStyle(.secondary) + } + } + } +} diff --git a/apps/shared/OpenClawKit/Sources/OpenClawChatUI/ChatMessageViews.swift b/apps/shared/OpenClawKit/Sources/OpenClawChatUI/ChatMessageViews.swift new file mode 100644 index 0000000000000000000000000000000000000000..baa790dbf747087f9d57acb778acde0ad38c889b --- /dev/null +++ b/apps/shared/OpenClawKit/Sources/OpenClawChatUI/ChatMessageViews.swift @@ -0,0 +1,616 @@ +import OpenClawKit +import Foundation +import SwiftUI + +private enum ChatUIConstants { + static let bubbleMaxWidth: CGFloat = 560 + static let bubbleCorner: CGFloat = 18 +} + +private struct ChatBubbleShape: InsettableShape { + enum Tail { + case left + case right + case none + } + + let cornerRadius: CGFloat + let tail: Tail + var insetAmount: CGFloat = 0 + + private let tailWidth: CGFloat = 7 + private let tailBaseHeight: CGFloat = 9 + + func inset(by amount: CGFloat) -> ChatBubbleShape { + var copy = self + copy.insetAmount += amount + return copy + } + + func path(in rect: CGRect) -> Path { + let rect = rect.insetBy(dx: self.insetAmount, dy: self.insetAmount) + switch self.tail { + case .left: + return self.leftTailPath(in: rect, radius: self.cornerRadius) + case .right: + return self.rightTailPath(in: rect, radius: self.cornerRadius) + case .none: + return Path(roundedRect: rect, cornerRadius: self.cornerRadius) + } + } + + private func rightTailPath(in rect: CGRect, radius r: CGFloat) -> Path { + var path = Path() + let bubbleMinX = rect.minX + let bubbleMaxX = rect.maxX - self.tailWidth + let bubbleMinY = rect.minY + let bubbleMaxY = rect.maxY + + let available = max(4, bubbleMaxY - bubbleMinY - 2 * r) + let baseH = min(tailBaseHeight, available) + let baseBottomY = bubbleMaxY - max(r * 0.45, 6) + let baseTopY = baseBottomY - baseH + let midY = (baseTopY + baseBottomY) / 2 + + let baseTop = CGPoint(x: bubbleMaxX, y: baseTopY) + let baseBottom = CGPoint(x: bubbleMaxX, y: baseBottomY) + let tip = CGPoint(x: bubbleMaxX + self.tailWidth, y: midY) + + path.move(to: CGPoint(x: bubbleMinX + r, y: bubbleMinY)) + path.addLine(to: CGPoint(x: bubbleMaxX - r, y: bubbleMinY)) + path.addQuadCurve( + to: CGPoint(x: bubbleMaxX, y: bubbleMinY + r), + control: CGPoint(x: bubbleMaxX, y: bubbleMinY)) + path.addLine(to: baseTop) + path.addCurve( + to: tip, + control1: CGPoint(x: bubbleMaxX + self.tailWidth * 0.2, y: baseTopY + baseH * 0.05), + control2: CGPoint(x: bubbleMaxX + self.tailWidth * 0.95, y: midY - baseH * 0.15)) + path.addCurve( + to: baseBottom, + control1: CGPoint(x: bubbleMaxX + self.tailWidth * 0.95, y: midY + baseH * 0.15), + control2: CGPoint(x: bubbleMaxX + self.tailWidth * 0.2, y: baseBottomY - baseH * 0.05)) + path.addQuadCurve( + to: CGPoint(x: bubbleMaxX - r, y: bubbleMaxY), + control: CGPoint(x: bubbleMaxX, y: bubbleMaxY)) + path.addLine(to: CGPoint(x: bubbleMinX + r, y: bubbleMaxY)) + path.addQuadCurve( + to: CGPoint(x: bubbleMinX, y: bubbleMaxY - r), + control: CGPoint(x: bubbleMinX, y: bubbleMaxY)) + path.addLine(to: CGPoint(x: bubbleMinX, y: bubbleMinY + r)) + path.addQuadCurve( + to: CGPoint(x: bubbleMinX + r, y: bubbleMinY), + control: CGPoint(x: bubbleMinX, y: bubbleMinY)) + + return path + } + + private func leftTailPath(in rect: CGRect, radius r: CGFloat) -> Path { + var path = Path() + let bubbleMinX = rect.minX + self.tailWidth + let bubbleMaxX = rect.maxX + let bubbleMinY = rect.minY + let bubbleMaxY = rect.maxY + + let available = max(4, bubbleMaxY - bubbleMinY - 2 * r) + let baseH = min(tailBaseHeight, available) + let baseBottomY = bubbleMaxY - max(r * 0.45, 6) + let baseTopY = baseBottomY - baseH + let midY = (baseTopY + baseBottomY) / 2 + + let baseTop = CGPoint(x: bubbleMinX, y: baseTopY) + let baseBottom = CGPoint(x: bubbleMinX, y: baseBottomY) + let tip = CGPoint(x: bubbleMinX - self.tailWidth, y: midY) + + path.move(to: CGPoint(x: bubbleMinX + r, y: bubbleMinY)) + path.addLine(to: CGPoint(x: bubbleMaxX - r, y: bubbleMinY)) + path.addQuadCurve( + to: CGPoint(x: bubbleMaxX, y: bubbleMinY + r), + control: CGPoint(x: bubbleMaxX, y: bubbleMinY)) + path.addLine(to: CGPoint(x: bubbleMaxX, y: bubbleMaxY - r)) + path.addQuadCurve( + to: CGPoint(x: bubbleMaxX - r, y: bubbleMaxY), + control: CGPoint(x: bubbleMaxX, y: bubbleMaxY)) + path.addLine(to: CGPoint(x: bubbleMinX + r, y: bubbleMaxY)) + path.addQuadCurve( + to: CGPoint(x: bubbleMinX, y: bubbleMaxY - r), + control: CGPoint(x: bubbleMinX, y: bubbleMaxY)) + path.addLine(to: baseBottom) + path.addCurve( + to: tip, + control1: CGPoint(x: bubbleMinX - self.tailWidth * 0.2, y: baseBottomY - baseH * 0.05), + control2: CGPoint(x: bubbleMinX - self.tailWidth * 0.95, y: midY + baseH * 0.15)) + path.addCurve( + to: baseTop, + control1: CGPoint(x: bubbleMinX - self.tailWidth * 0.95, y: midY - baseH * 0.15), + control2: CGPoint(x: bubbleMinX - self.tailWidth * 0.2, y: baseTopY + baseH * 0.05)) + path.addLine(to: CGPoint(x: bubbleMinX, y: bubbleMinY + r)) + path.addQuadCurve( + to: CGPoint(x: bubbleMinX + r, y: bubbleMinY), + control: CGPoint(x: bubbleMinX, y: bubbleMinY)) + + return path + } +} + +@MainActor +struct ChatMessageBubble: View { + let message: OpenClawChatMessage + let style: OpenClawChatView.Style + let markdownVariant: ChatMarkdownVariant + let userAccent: Color? + + var body: some View { + ChatMessageBody( + message: self.message, + isUser: self.isUser, + style: self.style, + markdownVariant: self.markdownVariant, + userAccent: self.userAccent) + .frame(maxWidth: ChatUIConstants.bubbleMaxWidth, alignment: self.isUser ? .trailing : .leading) + .frame(maxWidth: .infinity, alignment: self.isUser ? .trailing : .leading) + .padding(.horizontal, 2) + } + + private var isUser: Bool { self.message.role.lowercased() == "user" } +} + +@MainActor +private struct ChatMessageBody: View { + let message: OpenClawChatMessage + let isUser: Bool + let style: OpenClawChatView.Style + let markdownVariant: ChatMarkdownVariant + let userAccent: Color? + + var body: some View { + let text = self.primaryText + let textColor = self.isUser ? OpenClawChatTheme.userText : OpenClawChatTheme.assistantText + + VStack(alignment: .leading, spacing: 10) { + if self.isToolResultMessage { + if !text.isEmpty { + ToolResultCard( + title: self.toolResultTitle, + text: text, + isUser: self.isUser) + } + } else if self.isUser { + ChatMarkdownRenderer( + text: text, + context: .user, + variant: self.markdownVariant, + font: .system(size: 14), + textColor: textColor) + } else { + ChatAssistantTextBody(text: text, markdownVariant: self.markdownVariant) + } + + if !self.inlineAttachments.isEmpty { + ForEach(self.inlineAttachments.indices, id: \.self) { idx in + AttachmentRow(att: self.inlineAttachments[idx], isUser: self.isUser) + } + } + + if !self.toolCalls.isEmpty { + ForEach(self.toolCalls.indices, id: \.self) { idx in + ToolCallCard( + content: self.toolCalls[idx], + isUser: self.isUser) + } + } + + if !self.inlineToolResults.isEmpty { + ForEach(self.inlineToolResults.indices, id: \.self) { idx in + let toolResult = self.inlineToolResults[idx] + let display = ToolDisplayRegistry.resolve(name: toolResult.name ?? "tool", args: nil) + ToolResultCard( + title: "\(display.emoji) \(display.title)", + text: toolResult.text ?? "", + isUser: self.isUser) + } + } + } + .textSelection(.enabled) + .padding(.vertical, 10) + .padding(.horizontal, 12) + .foregroundStyle(textColor) + .background(self.bubbleBackground) + .clipShape(self.bubbleShape) + .overlay(self.bubbleBorder) + .shadow(color: self.bubbleShadowColor, radius: self.bubbleShadowRadius, y: self.bubbleShadowYOffset) + .padding(.leading, self.tailPaddingLeading) + .padding(.trailing, self.tailPaddingTrailing) + } + + private var primaryText: String { + let parts = self.message.content.compactMap { content -> String? in + let kind = (content.type ?? "text").lowercased() + guard kind == "text" || kind.isEmpty else { return nil } + return content.text + } + return parts.joined(separator: "\n").trimmingCharacters(in: .whitespacesAndNewlines) + } + + private var inlineAttachments: [OpenClawChatMessageContent] { + self.message.content.filter { content in + switch content.type ?? "text" { + case "file", "attachment": + true + default: + false + } + } + } + + private var toolCalls: [OpenClawChatMessageContent] { + self.message.content.filter { content in + let kind = (content.type ?? "").lowercased() + if ["toolcall", "tool_call", "tooluse", "tool_use"].contains(kind) { + return true + } + return content.name != nil && content.arguments != nil + } + } + + private var inlineToolResults: [OpenClawChatMessageContent] { + self.message.content.filter { content in + let kind = (content.type ?? "").lowercased() + return kind == "toolresult" || kind == "tool_result" + } + } + + private var isToolResultMessage: Bool { + let role = self.message.role.lowercased() + return role == "toolresult" || role == "tool_result" + } + + private var toolResultTitle: String { + if let name = self.message.toolName, !name.isEmpty { + let display = ToolDisplayRegistry.resolve(name: name, args: nil) + return "\(display.emoji) \(display.title)" + } + let display = ToolDisplayRegistry.resolve(name: "tool", args: nil) + return "\(display.emoji) \(display.title)" + } + + private var bubbleFillColor: Color { + if self.isUser { + return self.userAccent ?? OpenClawChatTheme.userBubble + } + if self.style == .onboarding { + return OpenClawChatTheme.onboardingAssistantBubble + } + return OpenClawChatTheme.assistantBubble + } + + private var bubbleBackground: AnyShapeStyle { + AnyShapeStyle(self.bubbleFillColor) + } + + private var bubbleBorderColor: Color { + if self.isUser { + return Color.white.opacity(0.12) + } + if self.style == .onboarding { + return OpenClawChatTheme.onboardingAssistantBorder + } + return Color.white.opacity(0.08) + } + + private var bubbleBorderWidth: CGFloat { + if self.isUser { return 0.5 } + if self.style == .onboarding { return 0.8 } + return 1 + } + + private var bubbleBorder: some View { + self.bubbleShape.strokeBorder(self.bubbleBorderColor, lineWidth: self.bubbleBorderWidth) + } + + private var bubbleShape: ChatBubbleShape { + ChatBubbleShape(cornerRadius: ChatUIConstants.bubbleCorner, tail: self.bubbleTail) + } + + private var bubbleTail: ChatBubbleShape.Tail { + guard self.style == .onboarding else { return .none } + return self.isUser ? .right : .left + } + + private var tailPaddingLeading: CGFloat { + self.style == .onboarding && !self.isUser ? 8 : 0 + } + + private var tailPaddingTrailing: CGFloat { + self.style == .onboarding && self.isUser ? 8 : 0 + } + + private var bubbleShadowColor: Color { + self.style == .onboarding && !self.isUser ? Color.black.opacity(0.28) : .clear + } + + private var bubbleShadowRadius: CGFloat { + self.style == .onboarding && !self.isUser ? 6 : 0 + } + + private var bubbleShadowYOffset: CGFloat { + self.style == .onboarding && !self.isUser ? 2 : 0 + } +} + +private struct AttachmentRow: View { + let att: OpenClawChatMessageContent + let isUser: Bool + + var body: some View { + HStack(spacing: 8) { + Image(systemName: "paperclip") + Text(self.att.fileName ?? "Attachment") + .font(.footnote) + .lineLimit(1) + .foregroundStyle(self.isUser ? OpenClawChatTheme.userText : OpenClawChatTheme.assistantText) + Spacer() + } + .padding(10) + .background(self.isUser ? Color.white.opacity(0.2) : Color.black.opacity(0.04)) + .clipShape(RoundedRectangle(cornerRadius: 12, style: .continuous)) + } +} + +private struct ToolCallCard: View { + let content: OpenClawChatMessageContent + let isUser: Bool + + var body: some View { + VStack(alignment: .leading, spacing: 6) { + HStack(spacing: 6) { + Text(self.toolName) + .font(.footnote.weight(.semibold)) + Spacer(minLength: 0) + } + + if let summary = self.summary, !summary.isEmpty { + Text(summary) + .font(.footnote.monospaced()) + .foregroundStyle(.secondary) + .lineLimit(2) + } + } + .padding(10) + .background( + RoundedRectangle(cornerRadius: 12, style: .continuous) + .fill(OpenClawChatTheme.subtleCard) + .overlay( + RoundedRectangle(cornerRadius: 12, style: .continuous) + .strokeBorder(Color.white.opacity(0.08), lineWidth: 1))) + } + + private var toolName: String { + "\(self.display.emoji) \(self.display.title)" + } + + private var summary: String? { + self.display.detailLine + } + + private var display: ToolDisplaySummary { + ToolDisplayRegistry.resolve(name: self.content.name ?? "tool", args: self.content.arguments) + } +} + +private struct ToolResultCard: View { + let title: String + let text: String + let isUser: Bool + @State private var expanded = false + + var body: some View { + VStack(alignment: .leading, spacing: 8) { + HStack(spacing: 6) { + Text(self.title) + .font(.footnote.weight(.semibold)) + Spacer(minLength: 0) + } + + Text(self.displayText) + .font(.footnote.monospaced()) + .foregroundStyle(self.isUser ? OpenClawChatTheme.userText : OpenClawChatTheme.assistantText) + .lineLimit(self.expanded ? nil : Self.previewLineLimit) + + if self.shouldShowToggle { + Button(self.expanded ? "Show less" : "Show full output") { + self.expanded.toggle() + } + .buttonStyle(.plain) + .font(.caption) + .foregroundStyle(.secondary) + } + } + .padding(10) + .background( + RoundedRectangle(cornerRadius: 12, style: .continuous) + .fill(OpenClawChatTheme.subtleCard) + .overlay( + RoundedRectangle(cornerRadius: 12, style: .continuous) + .strokeBorder(Color.white.opacity(0.08), lineWidth: 1))) + } + + private static let previewLineLimit = 8 + + private var lines: [Substring] { + self.text.components(separatedBy: .newlines).map { Substring($0) } + } + + private var displayText: String { + guard !self.expanded, self.lines.count > Self.previewLineLimit else { return self.text } + return self.lines.prefix(Self.previewLineLimit).joined(separator: "\n") + "\n…" + } + + private var shouldShowToggle: Bool { + self.lines.count > Self.previewLineLimit + } +} + +@MainActor +struct ChatTypingIndicatorBubble: View { + let style: OpenClawChatView.Style + + var body: some View { + HStack(spacing: 10) { + TypingDots() + if self.style == .standard { + Text("OpenClaw is thinking…") + .font(.subheadline) + .foregroundStyle(.secondary) + Spacer() + } + } + .padding(.vertical, self.style == .standard ? 12 : 10) + .padding(.horizontal, self.style == .standard ? 12 : 14) + .background( + RoundedRectangle(cornerRadius: 16, style: .continuous) + .fill(OpenClawChatTheme.assistantBubble)) + .overlay( + RoundedRectangle(cornerRadius: 16, style: .continuous) + .strokeBorder(Color.white.opacity(0.08), lineWidth: 1)) + .frame(maxWidth: ChatUIConstants.bubbleMaxWidth, alignment: .leading) + .focusable(false) + } +} + +extension ChatTypingIndicatorBubble: @MainActor Equatable { + static func == (lhs: Self, rhs: Self) -> Bool { + lhs.style == rhs.style + } +} + +@MainActor +struct ChatStreamingAssistantBubble: View { + let text: String + let markdownVariant: ChatMarkdownVariant + + var body: some View { + VStack(alignment: .leading, spacing: 10) { + ChatAssistantTextBody(text: self.text, markdownVariant: self.markdownVariant) + } + .padding(12) + .background( + RoundedRectangle(cornerRadius: 16, style: .continuous) + .fill(OpenClawChatTheme.assistantBubble)) + .overlay( + RoundedRectangle(cornerRadius: 16, style: .continuous) + .strokeBorder(Color.white.opacity(0.08), lineWidth: 1)) + .frame(maxWidth: ChatUIConstants.bubbleMaxWidth, alignment: .leading) + .focusable(false) + } +} + +@MainActor +struct ChatPendingToolsBubble: View { + let toolCalls: [OpenClawChatPendingToolCall] + + var body: some View { + VStack(alignment: .leading, spacing: 8) { + Label("Running tools…", systemImage: "hammer") + .font(.caption) + .foregroundStyle(.secondary) + + ForEach(self.toolCalls) { call in + let display = ToolDisplayRegistry.resolve(name: call.name, args: call.args) + VStack(alignment: .leading, spacing: 4) { + HStack(alignment: .firstTextBaseline, spacing: 8) { + Text("\(display.emoji) \(display.label)") + .font(.footnote.monospaced()) + .lineLimit(1) + Spacer(minLength: 0) + ProgressView().controlSize(.mini) + } + if let detail = display.detailLine, !detail.isEmpty { + Text(detail) + .font(.caption.monospaced()) + .foregroundStyle(.secondary) + .lineLimit(2) + } + } + .padding(10) + .background(Color.white.opacity(0.06)) + .clipShape(RoundedRectangle(cornerRadius: 12, style: .continuous)) + } + } + .padding(12) + .background( + RoundedRectangle(cornerRadius: 16, style: .continuous) + .fill(OpenClawChatTheme.assistantBubble)) + .overlay( + RoundedRectangle(cornerRadius: 16, style: .continuous) + .strokeBorder(Color.white.opacity(0.08), lineWidth: 1)) + .frame(maxWidth: ChatUIConstants.bubbleMaxWidth, alignment: .leading) + .focusable(false) + } +} + +extension ChatPendingToolsBubble: @MainActor Equatable { + static func == (lhs: Self, rhs: Self) -> Bool { + lhs.toolCalls == rhs.toolCalls + } +} + +@MainActor +private struct TypingDots: View { + @Environment(\.accessibilityReduceMotion) private var reduceMotion + @Environment(\.scenePhase) private var scenePhase + @State private var animate = false + + var body: some View { + HStack(spacing: 5) { + ForEach(0..<3, id: \.self) { idx in + Circle() + .fill(Color.secondary.opacity(0.55)) + .frame(width: 7, height: 7) + .scaleEffect(self.reduceMotion ? 0.85 : (self.animate ? 1.05 : 0.70)) + .opacity(self.reduceMotion ? 0.55 : (self.animate ? 0.95 : 0.30)) + .animation( + self.reduceMotion ? nil : .easeInOut(duration: 0.55) + .repeatForever(autoreverses: true) + .delay(Double(idx) * 0.16), + value: self.animate) + } + } + .onAppear { self.updateAnimationState() } + .onDisappear { self.animate = false } + .onChange(of: self.scenePhase) { _, _ in + self.updateAnimationState() + } + .onChange(of: self.reduceMotion) { _, _ in + self.updateAnimationState() + } + } + + private func updateAnimationState() { + guard !self.reduceMotion, self.scenePhase == .active else { + self.animate = false + return + } + self.animate = true + } +} + +private struct ChatAssistantTextBody: View { + let text: String + let markdownVariant: ChatMarkdownVariant + + var body: some View { + let segments = AssistantTextParser.segments(from: self.text) + VStack(alignment: .leading, spacing: 10) { + ForEach(segments) { segment in + let font = segment.kind == .thinking ? Font.system(size: 14).italic() : Font.system(size: 14) + ChatMarkdownRenderer( + text: segment.text, + context: .assistant, + variant: self.markdownVariant, + font: font, + textColor: OpenClawChatTheme.assistantText) + } + } + } +} diff --git a/apps/shared/OpenClawKit/Sources/OpenClawChatUI/ChatModels.swift b/apps/shared/OpenClawKit/Sources/OpenClawChatUI/ChatModels.swift new file mode 100644 index 0000000000000000000000000000000000000000..c58f2d702e48be6e5809b330130138d15671c49b --- /dev/null +++ b/apps/shared/OpenClawKit/Sources/OpenClawChatUI/ChatModels.swift @@ -0,0 +1,332 @@ +import OpenClawKit +import Foundation + +// NOTE: keep this file lightweight; decode must be resilient to varying transcript formats. + +#if canImport(AppKit) +import AppKit + +public typealias OpenClawPlatformImage = NSImage +#elseif canImport(UIKit) +import UIKit + +public typealias OpenClawPlatformImage = UIImage +#endif + +public struct OpenClawChatUsageCost: Codable, Hashable, Sendable { + public let input: Double? + public let output: Double? + public let cacheRead: Double? + public let cacheWrite: Double? + public let total: Double? +} + +public struct OpenClawChatUsage: Codable, Hashable, Sendable { + public let input: Int? + public let output: Int? + public let cacheRead: Int? + public let cacheWrite: Int? + public let cost: OpenClawChatUsageCost? + public let total: Int? + + enum CodingKeys: String, CodingKey { + case input + case output + case cacheRead + case cacheWrite + case cost + case total + case totalTokens + } + + public init(from decoder: Decoder) throws { + let container = try decoder.container(keyedBy: CodingKeys.self) + self.input = try container.decodeIfPresent(Int.self, forKey: .input) + self.output = try container.decodeIfPresent(Int.self, forKey: .output) + self.cacheRead = try container.decodeIfPresent(Int.self, forKey: .cacheRead) + self.cacheWrite = try container.decodeIfPresent(Int.self, forKey: .cacheWrite) + self.cost = try container.decodeIfPresent(OpenClawChatUsageCost.self, forKey: .cost) + self.total = + try container.decodeIfPresent(Int.self, forKey: .total) ?? + container.decodeIfPresent(Int.self, forKey: .totalTokens) + } + + public func encode(to encoder: Encoder) throws { + var container = encoder.container(keyedBy: CodingKeys.self) + try container.encodeIfPresent(self.input, forKey: .input) + try container.encodeIfPresent(self.output, forKey: .output) + try container.encodeIfPresent(self.cacheRead, forKey: .cacheRead) + try container.encodeIfPresent(self.cacheWrite, forKey: .cacheWrite) + try container.encodeIfPresent(self.cost, forKey: .cost) + try container.encodeIfPresent(self.total, forKey: .total) + } +} + +public struct OpenClawChatMessageContent: Codable, Hashable, Sendable { + public let type: String? + public let text: String? + public let thinking: String? + public let thinkingSignature: String? + public let mimeType: String? + public let fileName: String? + public let content: AnyCodable? + + // Tool-call fields (when `type == "toolCall"` or similar) + public let id: String? + public let name: String? + public let arguments: AnyCodable? + + public init( + type: String?, + text: String?, + thinking: String? = nil, + thinkingSignature: String? = nil, + mimeType: String?, + fileName: String?, + content: AnyCodable?, + id: String? = nil, + name: String? = nil, + arguments: AnyCodable? = nil) + { + self.type = type + self.text = text + self.thinking = thinking + self.thinkingSignature = thinkingSignature + self.mimeType = mimeType + self.fileName = fileName + self.content = content + self.id = id + self.name = name + self.arguments = arguments + } + + enum CodingKeys: String, CodingKey { + case type + case text + case thinking + case thinkingSignature + case mimeType + case fileName + case content + case id + case name + case arguments + } + + public init(from decoder: Decoder) throws { + let container = try decoder.container(keyedBy: CodingKeys.self) + self.type = try container.decodeIfPresent(String.self, forKey: .type) + self.text = try container.decodeIfPresent(String.self, forKey: .text) + self.thinking = try container.decodeIfPresent(String.self, forKey: .thinking) + self.thinkingSignature = try container.decodeIfPresent(String.self, forKey: .thinkingSignature) + self.mimeType = try container.decodeIfPresent(String.self, forKey: .mimeType) + self.fileName = try container.decodeIfPresent(String.self, forKey: .fileName) + self.id = try container.decodeIfPresent(String.self, forKey: .id) + self.name = try container.decodeIfPresent(String.self, forKey: .name) + self.arguments = try container.decodeIfPresent(AnyCodable.self, forKey: .arguments) + + if let any = try container.decodeIfPresent(AnyCodable.self, forKey: .content) { + self.content = any + } else if let str = try container.decodeIfPresent(String.self, forKey: .content) { + self.content = AnyCodable(str) + } else { + self.content = nil + } + } +} + +public struct OpenClawChatMessage: Codable, Identifiable, Sendable { + public var id: UUID = .init() + public let role: String + public let content: [OpenClawChatMessageContent] + public let timestamp: Double? + public let toolCallId: String? + public let toolName: String? + public let usage: OpenClawChatUsage? + public let stopReason: String? + + enum CodingKeys: String, CodingKey { + case role + case content + case timestamp + case toolCallId + case tool_call_id + case toolName + case tool_name + case usage + case stopReason + } + + public init( + id: UUID = .init(), + role: String, + content: [OpenClawChatMessageContent], + timestamp: Double?, + toolCallId: String? = nil, + toolName: String? = nil, + usage: OpenClawChatUsage? = nil, + stopReason: String? = nil) + { + self.id = id + self.role = role + self.content = content + self.timestamp = timestamp + self.toolCallId = toolCallId + self.toolName = toolName + self.usage = usage + self.stopReason = stopReason + } + + public init(from decoder: Decoder) throws { + let container = try decoder.container(keyedBy: CodingKeys.self) + self.role = try container.decode(String.self, forKey: .role) + self.timestamp = try container.decodeIfPresent(Double.self, forKey: .timestamp) + self.toolCallId = + try container.decodeIfPresent(String.self, forKey: .toolCallId) ?? + container.decodeIfPresent(String.self, forKey: .tool_call_id) + self.toolName = + try container.decodeIfPresent(String.self, forKey: .toolName) ?? + container.decodeIfPresent(String.self, forKey: .tool_name) + self.usage = try container.decodeIfPresent(OpenClawChatUsage.self, forKey: .usage) + self.stopReason = try container.decodeIfPresent(String.self, forKey: .stopReason) + + if let decoded = try? container.decode([OpenClawChatMessageContent].self, forKey: .content) { + self.content = decoded + return + } + + // Some session log formats store `content` as a plain string. + if let text = try? container.decode(String.self, forKey: .content) { + self.content = [ + OpenClawChatMessageContent( + type: "text", + text: text, + thinking: nil, + thinkingSignature: nil, + mimeType: nil, + fileName: nil, + content: nil, + id: nil, + name: nil, + arguments: nil), + ] + return + } + + self.content = [] + } + + public func encode(to encoder: Encoder) throws { + var container = encoder.container(keyedBy: CodingKeys.self) + try container.encode(self.role, forKey: .role) + try container.encodeIfPresent(self.timestamp, forKey: .timestamp) + try container.encodeIfPresent(self.toolCallId, forKey: .toolCallId) + try container.encodeIfPresent(self.toolName, forKey: .toolName) + try container.encodeIfPresent(self.usage, forKey: .usage) + try container.encodeIfPresent(self.stopReason, forKey: .stopReason) + try container.encode(self.content, forKey: .content) + } +} + +public struct OpenClawChatHistoryPayload: Codable, Sendable { + public let sessionKey: String + public let sessionId: String? + public let messages: [AnyCodable]? + public let thinkingLevel: String? +} + +public struct OpenClawSessionPreviewItem: Codable, Hashable, Sendable { + public let role: String + public let text: String +} + +public struct OpenClawSessionPreviewEntry: Codable, Sendable { + public let key: String + public let status: String + public let items: [OpenClawSessionPreviewItem] +} + +public struct OpenClawSessionsPreviewPayload: Codable, Sendable { + public let ts: Int + public let previews: [OpenClawSessionPreviewEntry] + + public init(ts: Int, previews: [OpenClawSessionPreviewEntry]) { + self.ts = ts + self.previews = previews + } +} + +public struct OpenClawChatSendResponse: Codable, Sendable { + public let runId: String + public let status: String +} + +public struct OpenClawChatEventPayload: Codable, Sendable { + public let runId: String? + public let sessionKey: String? + public let state: String? + public let message: AnyCodable? + public let errorMessage: String? +} + +public struct OpenClawAgentEventPayload: Codable, Sendable, Identifiable { + public var id: String { "\(self.runId)-\(self.seq ?? -1)" } + public let runId: String + public let seq: Int? + public let stream: String + public let ts: Int? + public let data: [String: AnyCodable] +} + +public struct OpenClawChatPendingToolCall: Identifiable, Hashable, Sendable { + public var id: String { self.toolCallId } + public let toolCallId: String + public let name: String + public let args: AnyCodable? + public let startedAt: Double? + public let isError: Bool? +} + +public struct OpenClawGatewayHealthOK: Codable, Sendable { + public let ok: Bool? +} + +public struct OpenClawPendingAttachment: Identifiable { + public let id = UUID() + public let url: URL? + public let data: Data + public let fileName: String + public let mimeType: String + public let type: String + public let preview: OpenClawPlatformImage? + + public init( + url: URL?, + data: Data, + fileName: String, + mimeType: String, + type: String = "file", + preview: OpenClawPlatformImage?) + { + self.url = url + self.data = data + self.fileName = fileName + self.mimeType = mimeType + self.type = type + self.preview = preview + } +} + +public struct OpenClawChatAttachmentPayload: Codable, Sendable, Hashable { + public let type: String + public let mimeType: String + public let fileName: String + public let content: String + + public init(type: String, mimeType: String, fileName: String, content: String) { + self.type = type + self.mimeType = mimeType + self.fileName = fileName + self.content = content + } +} diff --git a/apps/shared/OpenClawKit/Sources/OpenClawChatUI/ChatPayloadDecoding.swift b/apps/shared/OpenClawKit/Sources/OpenClawChatUI/ChatPayloadDecoding.swift new file mode 100644 index 0000000000000000000000000000000000000000..02636696d210f37c1a43803e6595fa328386de09 --- /dev/null +++ b/apps/shared/OpenClawKit/Sources/OpenClawChatUI/ChatPayloadDecoding.swift @@ -0,0 +1,9 @@ +import OpenClawKit +import Foundation + +enum ChatPayloadDecoding { + static func decode(_ payload: AnyCodable, as _: T.Type = T.self) throws -> T { + let data = try JSONEncoder().encode(payload) + return try JSONDecoder().decode(T.self, from: data) + } +} diff --git a/apps/shared/OpenClawKit/Sources/OpenClawChatUI/ChatSessions.swift b/apps/shared/OpenClawKit/Sources/OpenClawChatUI/ChatSessions.swift new file mode 100644 index 0000000000000000000000000000000000000000..febe69a3cbe3e03dd9adfa7e93e45af9e24860b6 --- /dev/null +++ b/apps/shared/OpenClawKit/Sources/OpenClawChatUI/ChatSessions.swift @@ -0,0 +1,40 @@ +import Foundation + +public struct OpenClawChatSessionsDefaults: Codable, Sendable { + public let model: String? + public let contextTokens: Int? +} + +public struct OpenClawChatSessionEntry: Codable, Identifiable, Sendable, Hashable { + public var id: String { self.key } + + public let key: String + public let kind: String? + public let displayName: String? + public let surface: String? + public let subject: String? + public let room: String? + public let space: String? + public let updatedAt: Double? + public let sessionId: String? + + public let systemSent: Bool? + public let abortedLastRun: Bool? + public let thinkingLevel: String? + public let verboseLevel: String? + + public let inputTokens: Int? + public let outputTokens: Int? + public let totalTokens: Int? + + public let model: String? + public let contextTokens: Int? +} + +public struct OpenClawChatSessionsListResponse: Codable, Sendable { + public let ts: Double? + public let path: String? + public let count: Int? + public let defaults: OpenClawChatSessionsDefaults? + public let sessions: [OpenClawChatSessionEntry] +} diff --git a/apps/shared/OpenClawKit/Sources/OpenClawChatUI/ChatSheets.swift b/apps/shared/OpenClawKit/Sources/OpenClawChatUI/ChatSheets.swift new file mode 100644 index 0000000000000000000000000000000000000000..678000d2cea44023a18b729f21860c0fe20fe1aa --- /dev/null +++ b/apps/shared/OpenClawKit/Sources/OpenClawChatUI/ChatSheets.swift @@ -0,0 +1,69 @@ +import Observation +import SwiftUI + +@MainActor +struct ChatSessionsSheet: View { + @Bindable var viewModel: OpenClawChatViewModel + @Environment(\.dismiss) private var dismiss + + var body: some View { + NavigationStack { + List(self.viewModel.sessions) { session in + Button { + self.viewModel.switchSession(to: session.key) + self.dismiss() + } label: { + VStack(alignment: .leading, spacing: 4) { + Text(session.displayName ?? session.key) + .font(.system(.body, design: .monospaced)) + .lineLimit(1) + if let updatedAt = session.updatedAt, updatedAt > 0 { + Text(Date(timeIntervalSince1970: updatedAt / 1000).formatted( + date: .abbreviated, + time: .shortened)) + .font(.caption) + .foregroundStyle(.secondary) + } + } + } + } + .navigationTitle("Sessions") + .toolbar { + #if os(macOS) + ToolbarItem(placement: .automatic) { + Button { + self.viewModel.refreshSessions(limit: 200) + } label: { + Image(systemName: "arrow.clockwise") + } + } + ToolbarItem(placement: .primaryAction) { + Button { + self.dismiss() + } label: { + Image(systemName: "xmark") + } + } + #else + ToolbarItem(placement: .topBarLeading) { + Button { + self.viewModel.refreshSessions(limit: 200) + } label: { + Image(systemName: "arrow.clockwise") + } + } + ToolbarItem(placement: .topBarTrailing) { + Button { + self.dismiss() + } label: { + Image(systemName: "xmark") + } + } + #endif + } + .onAppear { + self.viewModel.refreshSessions(limit: 200) + } + } + } +} diff --git a/apps/shared/OpenClawKit/Sources/OpenClawChatUI/ChatTheme.swift b/apps/shared/OpenClawKit/Sources/OpenClawChatUI/ChatTheme.swift new file mode 100644 index 0000000000000000000000000000000000000000..c06ed4f46af2919501f201dcc6f5d458484038a6 --- /dev/null +++ b/apps/shared/OpenClawKit/Sources/OpenClawChatUI/ChatTheme.swift @@ -0,0 +1,174 @@ +import SwiftUI + +#if os(macOS) +import AppKit +#else +import UIKit +#endif + +#if os(macOS) +extension NSAppearance { + fileprivate var isDarkAqua: Bool { + self.bestMatch(from: [.aqua, .darkAqua]) == .darkAqua + } +} +#endif + +enum OpenClawChatTheme { + #if os(macOS) + static func resolvedAssistantBubbleColor(for appearance: NSAppearance) -> NSColor { + // NSColor semantic colors don't reliably resolve for arbitrary NSAppearance in SwiftPM. + // Use explicit light/dark values so the bubble updates when the system appearance flips. + appearance.isDarkAqua + ? NSColor(calibratedWhite: 0.18, alpha: 0.88) + : NSColor(calibratedWhite: 0.94, alpha: 0.92) + } + + static func resolvedOnboardingAssistantBubbleColor(for appearance: NSAppearance) -> NSColor { + appearance.isDarkAqua + ? NSColor(calibratedWhite: 0.20, alpha: 0.94) + : NSColor(calibratedWhite: 0.97, alpha: 0.98) + } + + static let assistantBubbleDynamicNSColor = NSColor( + name: NSColor.Name("OpenClawChatTheme.assistantBubble"), + dynamicProvider: resolvedAssistantBubbleColor(for:)) + + static let onboardingAssistantBubbleDynamicNSColor = NSColor( + name: NSColor.Name("OpenClawChatTheme.onboardingAssistantBubble"), + dynamicProvider: resolvedOnboardingAssistantBubbleColor(for:)) + #endif + + static var surface: Color { + #if os(macOS) + Color(nsColor: .windowBackgroundColor) + #else + Color(uiColor: .systemBackground) + #endif + } + + @ViewBuilder + static var background: some View { + #if os(macOS) + ZStack { + Rectangle() + .fill(.ultraThinMaterial) + LinearGradient( + colors: [ + Color.white.opacity(0.12), + Color(nsColor: .windowBackgroundColor).opacity(0.35), + Color.black.opacity(0.35), + ], + startPoint: .topLeading, + endPoint: .bottomTrailing) + RadialGradient( + colors: [ + Color(nsColor: .systemOrange).opacity(0.14), + .clear, + ], + center: .topLeading, + startRadius: 40, + endRadius: 320) + RadialGradient( + colors: [ + Color(nsColor: .systemTeal).opacity(0.12), + .clear, + ], + center: .topTrailing, + startRadius: 40, + endRadius: 280) + Color.black.opacity(0.08) + } + #else + Color(uiColor: .systemBackground) + #endif + } + + static var card: Color { + #if os(macOS) + Color(nsColor: .textBackgroundColor) + #else + Color(uiColor: .secondarySystemBackground) + #endif + } + + static var subtleCard: AnyShapeStyle { + #if os(macOS) + AnyShapeStyle(.ultraThinMaterial) + #else + AnyShapeStyle(Color(uiColor: .secondarySystemBackground).opacity(0.9)) + #endif + } + + static var userBubble: Color { + Color(red: 127 / 255.0, green: 184 / 255.0, blue: 212 / 255.0) + } + + static var assistantBubble: Color { + #if os(macOS) + Color(nsColor: self.assistantBubbleDynamicNSColor) + #else + Color(uiColor: .secondarySystemBackground) + #endif + } + + static var onboardingAssistantBubble: Color { + #if os(macOS) + Color(nsColor: self.onboardingAssistantBubbleDynamicNSColor) + #else + Color(uiColor: .secondarySystemBackground) + #endif + } + + static var onboardingAssistantBorder: Color { + #if os(macOS) + Color.white.opacity(0.12) + #else + Color.white.opacity(0.12) + #endif + } + + static var userText: Color { .white } + + static var assistantText: Color { + #if os(macOS) + Color(nsColor: .labelColor) + #else + Color(uiColor: .label) + #endif + } + + static var composerBackground: AnyShapeStyle { + #if os(macOS) + AnyShapeStyle(.ultraThinMaterial) + #else + AnyShapeStyle(Color(uiColor: .systemBackground)) + #endif + } + + static var composerField: AnyShapeStyle { + #if os(macOS) + AnyShapeStyle(.thinMaterial) + #else + AnyShapeStyle(Color(uiColor: .secondarySystemBackground)) + #endif + } + + static var composerBorder: Color { + Color.white.opacity(0.12) + } + + static var divider: Color { + Color.secondary.opacity(0.2) + } +} + +enum OpenClawPlatformImageFactory { + static func image(_ image: OpenClawPlatformImage) -> Image { + #if os(macOS) + Image(nsImage: image) + #else + Image(uiImage: image) + #endif + } +} diff --git a/apps/shared/OpenClawKit/Sources/OpenClawChatUI/ChatTransport.swift b/apps/shared/OpenClawKit/Sources/OpenClawChatUI/ChatTransport.swift new file mode 100644 index 0000000000000000000000000000000000000000..037c1352205d6d4ac390fcc4a13a909f3b4c8345 --- /dev/null +++ b/apps/shared/OpenClawKit/Sources/OpenClawChatUI/ChatTransport.swift @@ -0,0 +1,45 @@ +import Foundation + +public enum OpenClawChatTransportEvent: Sendable { + case health(ok: Bool) + case tick + case chat(OpenClawChatEventPayload) + case agent(OpenClawAgentEventPayload) + case seqGap +} + +public protocol OpenClawChatTransport: Sendable { + func requestHistory(sessionKey: String) async throws -> OpenClawChatHistoryPayload + func sendMessage( + sessionKey: String, + message: String, + thinking: String, + idempotencyKey: String, + attachments: [OpenClawChatAttachmentPayload]) async throws -> OpenClawChatSendResponse + + func abortRun(sessionKey: String, runId: String) async throws + func listSessions(limit: Int?) async throws -> OpenClawChatSessionsListResponse + + func requestHealth(timeoutMs: Int) async throws -> Bool + func events() -> AsyncStream + + func setActiveSessionKey(_ sessionKey: String) async throws +} + +extension OpenClawChatTransport { + public func setActiveSessionKey(_: String) async throws {} + + public func abortRun(sessionKey _: String, runId _: String) async throws { + throw NSError( + domain: "OpenClawChatTransport", + code: 0, + userInfo: [NSLocalizedDescriptionKey: "chat.abort not supported by this transport"]) + } + + public func listSessions(limit _: Int?) async throws -> OpenClawChatSessionsListResponse { + throw NSError( + domain: "OpenClawChatTransport", + code: 0, + userInfo: [NSLocalizedDescriptionKey: "sessions.list not supported by this transport"]) + } +} diff --git a/apps/shared/OpenClawKit/Sources/OpenClawChatUI/ChatView.swift b/apps/shared/OpenClawKit/Sources/OpenClawChatUI/ChatView.swift new file mode 100644 index 0000000000000000000000000000000000000000..68f9ae2f311e721047727aa72a0e52dd2bcf4bb0 --- /dev/null +++ b/apps/shared/OpenClawKit/Sources/OpenClawChatUI/ChatView.swift @@ -0,0 +1,507 @@ +import SwiftUI + +@MainActor +public struct OpenClawChatView: View { + public enum Style { + case standard + case onboarding + } + + @State private var viewModel: OpenClawChatViewModel + @State private var scrollerBottomID = UUID() + @State private var scrollPosition: UUID? + @State private var showSessions = false + @State private var hasPerformedInitialScroll = false + @State private var isPinnedToBottom = true + @State private var lastUserMessageID: UUID? + private let showsSessionSwitcher: Bool + private let style: Style + private let markdownVariant: ChatMarkdownVariant + private let userAccent: Color? + + private enum Layout { + #if os(macOS) + static let outerPaddingHorizontal: CGFloat = 6 + static let outerPaddingVertical: CGFloat = 0 + static let composerPaddingHorizontal: CGFloat = 0 + static let stackSpacing: CGFloat = 0 + static let messageSpacing: CGFloat = 6 + static let messageListPaddingTop: CGFloat = 12 + static let messageListPaddingBottom: CGFloat = 16 + static let messageListPaddingHorizontal: CGFloat = 6 + #else + static let outerPaddingHorizontal: CGFloat = 6 + static let outerPaddingVertical: CGFloat = 6 + static let composerPaddingHorizontal: CGFloat = 6 + static let stackSpacing: CGFloat = 6 + static let messageSpacing: CGFloat = 12 + static let messageListPaddingTop: CGFloat = 10 + static let messageListPaddingBottom: CGFloat = 6 + static let messageListPaddingHorizontal: CGFloat = 8 + #endif + } + + public init( + viewModel: OpenClawChatViewModel, + showsSessionSwitcher: Bool = false, + style: Style = .standard, + markdownVariant: ChatMarkdownVariant = .standard, + userAccent: Color? = nil) + { + self._viewModel = State(initialValue: viewModel) + self.showsSessionSwitcher = showsSessionSwitcher + self.style = style + self.markdownVariant = markdownVariant + self.userAccent = userAccent + } + + public var body: some View { + ZStack { + if self.style == .standard { + OpenClawChatTheme.background + .ignoresSafeArea() + } + + VStack(spacing: Layout.stackSpacing) { + self.messageList + .padding(.horizontal, Layout.outerPaddingHorizontal) + OpenClawChatComposer( + viewModel: self.viewModel, + style: self.style, + showsSessionSwitcher: self.showsSessionSwitcher) + .padding(.horizontal, Layout.composerPaddingHorizontal) + } + .padding(.vertical, Layout.outerPaddingVertical) + .frame(maxWidth: .infinity) + .frame(maxHeight: .infinity, alignment: .top) + } + .frame(maxWidth: .infinity, maxHeight: .infinity, alignment: .top) + .onAppear { self.viewModel.load() } + .sheet(isPresented: self.$showSessions) { + if self.showsSessionSwitcher { + ChatSessionsSheet(viewModel: self.viewModel) + } else { + EmptyView() + } + } + } + + private var messageList: some View { + ZStack { + ScrollView { + LazyVStack(spacing: Layout.messageSpacing) { + self.messageListRows + + Color.clear + #if os(macOS) + .frame(height: Layout.messageListPaddingBottom) + #else + .frame(height: Layout.messageListPaddingBottom + 1) + #endif + .id(self.scrollerBottomID) + } + // Use scroll targets for stable auto-scroll without ScrollViewReader relayout glitches. + .scrollTargetLayout() + .padding(.top, Layout.messageListPaddingTop) + .padding(.horizontal, Layout.messageListPaddingHorizontal) + } + // Keep the scroll pinned to the bottom for new messages. + .scrollPosition(id: self.$scrollPosition, anchor: .bottom) + .onChange(of: self.scrollPosition) { _, position in + guard let position else { return } + self.isPinnedToBottom = position == self.scrollerBottomID + } + + if self.viewModel.isLoading { + ProgressView() + .controlSize(.large) + .frame(maxWidth: .infinity, maxHeight: .infinity) + } + + self.messageListOverlay + } + // Ensure the message list claims vertical space on the first layout pass. + .frame(maxHeight: .infinity, alignment: .top) + .layoutPriority(1) + .onChange(of: self.viewModel.isLoading) { _, isLoading in + guard !isLoading, !self.hasPerformedInitialScroll else { return } + self.scrollPosition = self.scrollerBottomID + self.hasPerformedInitialScroll = true + self.isPinnedToBottom = true + } + .onChange(of: self.viewModel.sessionKey) { _, _ in + self.hasPerformedInitialScroll = false + self.isPinnedToBottom = true + } + .onChange(of: self.viewModel.isSending) { _, isSending in + // Scroll to bottom when user sends a message, even if scrolled up. + guard isSending, self.hasPerformedInitialScroll else { return } + self.isPinnedToBottom = true + withAnimation(.snappy(duration: 0.22)) { + self.scrollPosition = self.scrollerBottomID + } + } + .onChange(of: self.viewModel.messages.count) { _, _ in + guard self.hasPerformedInitialScroll else { return } + if let lastMessage = self.viewModel.messages.last, + lastMessage.role.lowercased() == "user", + lastMessage.id != self.lastUserMessageID { + self.lastUserMessageID = lastMessage.id + self.isPinnedToBottom = true + withAnimation(.snappy(duration: 0.22)) { + self.scrollPosition = self.scrollerBottomID + } + return + } + + guard self.isPinnedToBottom else { return } + withAnimation(.snappy(duration: 0.22)) { + self.scrollPosition = self.scrollerBottomID + } + } + .onChange(of: self.viewModel.pendingRunCount) { _, _ in + guard self.hasPerformedInitialScroll, self.isPinnedToBottom else { return } + withAnimation(.snappy(duration: 0.22)) { + self.scrollPosition = self.scrollerBottomID + } + } + .onChange(of: self.viewModel.streamingAssistantText) { _, _ in + guard self.hasPerformedInitialScroll, self.isPinnedToBottom else { return } + withAnimation(.snappy(duration: 0.22)) { + self.scrollPosition = self.scrollerBottomID + } + } + } + + @ViewBuilder + private var messageListRows: some View { + ForEach(self.visibleMessages) { msg in + ChatMessageBubble( + message: msg, + style: self.style, + markdownVariant: self.markdownVariant, + userAccent: self.userAccent) + .frame( + maxWidth: .infinity, + alignment: msg.role.lowercased() == "user" ? .trailing : .leading) + } + + if self.viewModel.pendingRunCount > 0 { + HStack { + ChatTypingIndicatorBubble(style: self.style) + .equatable() + Spacer(minLength: 0) + } + } + + if !self.viewModel.pendingToolCalls.isEmpty { + ChatPendingToolsBubble(toolCalls: self.viewModel.pendingToolCalls) + .equatable() + .frame(maxWidth: .infinity, alignment: .leading) + } + + if let text = self.viewModel.streamingAssistantText, AssistantTextParser.hasVisibleContent(in: text) { + ChatStreamingAssistantBubble(text: text, markdownVariant: self.markdownVariant) + .frame(maxWidth: .infinity, alignment: .leading) + } + } + + private var visibleMessages: [OpenClawChatMessage] { + let base: [OpenClawChatMessage] + if self.style == .onboarding { + guard let first = self.viewModel.messages.first else { return [] } + base = first.role.lowercased() == "user" ? Array(self.viewModel.messages.dropFirst()) : self.viewModel + .messages + } else { + base = self.viewModel.messages + } + return self.mergeToolResults(in: base) + } + + @ViewBuilder + private var messageListOverlay: some View { + if self.viewModel.isLoading { + EmptyView() + } else if let error = self.activeErrorText { + let presentation = self.errorPresentation(for: error) + if self.hasVisibleMessageListContent { + VStack(spacing: 0) { + ChatNoticeBanner( + systemImage: presentation.systemImage, + title: presentation.title, + message: error, + tint: presentation.tint, + dismiss: { self.viewModel.errorText = nil }, + refresh: { self.viewModel.refresh() }) + Spacer(minLength: 0) + } + .padding(.horizontal, 10) + .padding(.top, 8) + .frame(maxWidth: .infinity, maxHeight: .infinity, alignment: .top) + } else { + ChatNoticeCard( + systemImage: presentation.systemImage, + title: presentation.title, + message: error, + tint: presentation.tint, + actionTitle: "Refresh", + action: { self.viewModel.refresh() }) + .padding(.horizontal, 24) + .frame(maxWidth: .infinity, maxHeight: .infinity) + } + } else if self.showsEmptyState { + ChatNoticeCard( + systemImage: "bubble.left.and.bubble.right.fill", + title: self.emptyStateTitle, + message: self.emptyStateMessage, + tint: .accentColor, + actionTitle: nil, + action: nil) + .padding(.horizontal, 24) + .frame(maxWidth: .infinity, maxHeight: .infinity) + } + } + + private var activeErrorText: String? { + guard let text = self.viewModel.errorText? + .trimmingCharacters(in: .whitespacesAndNewlines), + !text.isEmpty + else { + return nil + } + return text + } + + private var hasVisibleMessageListContent: Bool { + if !self.visibleMessages.isEmpty { + return true + } + if let text = self.viewModel.streamingAssistantText, + AssistantTextParser.hasVisibleContent(in: text) + { + return true + } + if self.viewModel.pendingRunCount > 0 { + return true + } + if !self.viewModel.pendingToolCalls.isEmpty { + return true + } + return false + } + + private var showsEmptyState: Bool { + self.viewModel.messages.isEmpty && + !(self.viewModel.streamingAssistantText.map { AssistantTextParser.hasVisibleContent(in: $0) } ?? false) && + self.viewModel.pendingRunCount == 0 && + self.viewModel.pendingToolCalls.isEmpty + } + + private var emptyStateTitle: String { + #if os(macOS) + "Web Chat" + #else + "Chat" + #endif + } + + private var emptyStateMessage: String { + #if os(macOS) + "Type a message below to start.\nReturn sends • Shift-Return adds a line break." + #else + "Type a message below to start." + #endif + } + + private func errorPresentation(for error: String) -> (title: String, systemImage: String, tint: Color) { + let lower = error.lowercased() + if lower.contains("not connected") || lower.contains("socket") { + return ("Disconnected", "wifi.slash", .orange) + } + if lower.contains("timed out") { + return ("Timed out", "clock.badge.exclamationmark", .orange) + } + return ("Error", "exclamationmark.triangle.fill", .orange) + } + + private func mergeToolResults(in messages: [OpenClawChatMessage]) -> [OpenClawChatMessage] { + var result: [OpenClawChatMessage] = [] + result.reserveCapacity(messages.count) + + for message in messages { + guard self.isToolResultMessage(message) else { + result.append(message) + continue + } + + guard let toolCallId = message.toolCallId, + let last = result.last, + self.toolCallIds(in: last).contains(toolCallId) + else { + result.append(message) + continue + } + + let toolText = self.toolResultText(from: message) + if toolText.isEmpty { + continue + } + + var content = last.content + content.append( + OpenClawChatMessageContent( + type: "tool_result", + text: toolText, + thinking: nil, + thinkingSignature: nil, + mimeType: nil, + fileName: nil, + content: nil, + id: toolCallId, + name: message.toolName, + arguments: nil)) + + let merged = OpenClawChatMessage( + id: last.id, + role: last.role, + content: content, + timestamp: last.timestamp, + toolCallId: last.toolCallId, + toolName: last.toolName, + usage: last.usage, + stopReason: last.stopReason) + result[result.count - 1] = merged + } + + return result + } + + private func isToolResultMessage(_ message: OpenClawChatMessage) -> Bool { + let role = message.role.lowercased() + return role == "toolresult" || role == "tool_result" + } + + private func toolCallIds(in message: OpenClawChatMessage) -> Set { + var ids = Set() + for content in message.content { + let kind = (content.type ?? "").lowercased() + let isTool = + ["toolcall", "tool_call", "tooluse", "tool_use"].contains(kind) || + (content.name != nil && content.arguments != nil) + if isTool, let id = content.id { + ids.insert(id) + } + } + if let toolCallId = message.toolCallId { + ids.insert(toolCallId) + } + return ids + } + + private func toolResultText(from message: OpenClawChatMessage) -> String { + let parts = message.content.compactMap { content -> String? in + let kind = (content.type ?? "text").lowercased() + guard kind == "text" || kind.isEmpty else { return nil } + return content.text + } + return parts.joined(separator: "\n").trimmingCharacters(in: .whitespacesAndNewlines) + } +} + +private struct ChatNoticeCard: View { + let systemImage: String + let title: String + let message: String + let tint: Color + let actionTitle: String? + let action: (() -> Void)? + + var body: some View { + VStack(spacing: 12) { + ZStack { + Circle() + .fill(self.tint.opacity(0.16)) + Image(systemName: self.systemImage) + .font(.system(size: 24, weight: .semibold)) + .foregroundStyle(self.tint) + } + .frame(width: 52, height: 52) + + Text(self.title) + .font(.headline) + + Text(self.message) + .font(.callout) + .foregroundStyle(.secondary) + .multilineTextAlignment(.center) + .lineLimit(4) + .frame(maxWidth: 360) + + if let actionTitle, let action { + Button(actionTitle, action: action) + .buttonStyle(.borderedProminent) + .controlSize(.small) + } + } + .padding(18) + .background( + RoundedRectangle(cornerRadius: 18, style: .continuous) + .fill(OpenClawChatTheme.subtleCard) + .overlay( + RoundedRectangle(cornerRadius: 18, style: .continuous) + .strokeBorder(Color.white.opacity(0.12), lineWidth: 1))) + .shadow(color: .black.opacity(0.14), radius: 18, y: 8) + } +} + +private struct ChatNoticeBanner: View { + let systemImage: String + let title: String + let message: String + let tint: Color + let dismiss: () -> Void + let refresh: () -> Void + + var body: some View { + HStack(alignment: .top, spacing: 10) { + Image(systemName: self.systemImage) + .font(.system(size: 15, weight: .semibold)) + .foregroundStyle(self.tint) + .padding(.top, 1) + + VStack(alignment: .leading, spacing: 3) { + Text(self.title) + .font(.caption.weight(.semibold)) + + Text(self.message) + .font(.caption) + .foregroundStyle(.secondary) + .lineLimit(2) + } + + Spacer(minLength: 0) + + Button(action: self.refresh) { + Image(systemName: "arrow.clockwise") + } + .buttonStyle(.bordered) + .controlSize(.small) + .help("Refresh") + + Button(action: self.dismiss) { + Image(systemName: "xmark") + } + .buttonStyle(.plain) + .foregroundStyle(.secondary) + .help("Dismiss") + } + .padding(.horizontal, 12) + .padding(.vertical, 10) + .background( + RoundedRectangle(cornerRadius: 14, style: .continuous) + .fill(OpenClawChatTheme.subtleCard) + .overlay( + RoundedRectangle(cornerRadius: 14, style: .continuous) + .strokeBorder(Color.white.opacity(0.12), lineWidth: 1))) + } +} diff --git a/apps/shared/OpenClawKit/Sources/OpenClawChatUI/ChatViewModel.swift b/apps/shared/OpenClawKit/Sources/OpenClawChatUI/ChatViewModel.swift new file mode 100644 index 0000000000000000000000000000000000000000..272fd81c11dfec56f1dc1f392501afe152a3d9dd --- /dev/null +++ b/apps/shared/OpenClawKit/Sources/OpenClawChatUI/ChatViewModel.swift @@ -0,0 +1,554 @@ +import OpenClawKit +import Foundation +import Observation +import OSLog +import UniformTypeIdentifiers + +#if canImport(AppKit) +import AppKit +#elseif canImport(UIKit) +import UIKit +#endif + +private let chatUILogger = Logger(subsystem: "ai.openclaw", category: "OpenClawChatUI") + +@MainActor +@Observable +public final class OpenClawChatViewModel { + public private(set) var messages: [OpenClawChatMessage] = [] + public var input: String = "" + public var thinkingLevel: String = "off" + public private(set) var isLoading = false + public private(set) var isSending = false + public private(set) var isAborting = false + public var errorText: String? + public var attachments: [OpenClawPendingAttachment] = [] + public private(set) var healthOK: Bool = false + public private(set) var pendingRunCount: Int = 0 + + public private(set) var sessionKey: String + public private(set) var sessionId: String? + public private(set) var streamingAssistantText: String? + public private(set) var pendingToolCalls: [OpenClawChatPendingToolCall] = [] + public private(set) var sessions: [OpenClawChatSessionEntry] = [] + private let transport: any OpenClawChatTransport + + @ObservationIgnored + private nonisolated(unsafe) var eventTask: Task? + private var pendingRuns = Set() { + didSet { self.pendingRunCount = self.pendingRuns.count } + } + + @ObservationIgnored + private nonisolated(unsafe) var pendingRunTimeoutTasks: [String: Task] = [:] + private let pendingRunTimeoutMs: UInt64 = 120_000 + + private var pendingToolCallsById: [String: OpenClawChatPendingToolCall] = [:] { + didSet { + self.pendingToolCalls = self.pendingToolCallsById.values + .sorted { ($0.startedAt ?? 0) < ($1.startedAt ?? 0) } + } + } + + private var lastHealthPollAt: Date? + + public init(sessionKey: String, transport: any OpenClawChatTransport) { + self.sessionKey = sessionKey + self.transport = transport + + self.eventTask = Task { [weak self] in + guard let self else { return } + let stream = self.transport.events() + for await evt in stream { + if Task.isCancelled { return } + await MainActor.run { [weak self] in + self?.handleTransportEvent(evt) + } + } + } + } + + deinit { + self.eventTask?.cancel() + for (_, task) in self.pendingRunTimeoutTasks { + task.cancel() + } + } + + public func load() { + Task { await self.bootstrap() } + } + + public func refresh() { + Task { await self.bootstrap() } + } + + public func send() { + Task { await self.performSend() } + } + + public func abort() { + Task { await self.performAbort() } + } + + public func refreshSessions(limit: Int? = nil) { + Task { await self.fetchSessions(limit: limit) } + } + + public func switchSession(to sessionKey: String) { + Task { await self.performSwitchSession(to: sessionKey) } + } + + public var sessionChoices: [OpenClawChatSessionEntry] { + let now = Date().timeIntervalSince1970 * 1000 + let cutoff = now - (24 * 60 * 60 * 1000) + let sorted = self.sessions.sorted { ($0.updatedAt ?? 0) > ($1.updatedAt ?? 0) } + var seen = Set() + var recent: [OpenClawChatSessionEntry] = [] + for entry in sorted { + guard !seen.contains(entry.key) else { continue } + seen.insert(entry.key) + guard (entry.updatedAt ?? 0) >= cutoff else { continue } + recent.append(entry) + } + + var result: [OpenClawChatSessionEntry] = [] + var included = Set() + for entry in recent where !included.contains(entry.key) { + result.append(entry) + included.insert(entry.key) + } + + if !included.contains(self.sessionKey) { + if let current = sorted.first(where: { $0.key == self.sessionKey }) { + result.append(current) + } else { + result.append(self.placeholderSession(key: self.sessionKey)) + } + } + + return result + } + + public func addAttachments(urls: [URL]) { + Task { await self.loadAttachments(urls: urls) } + } + + public func addImageAttachment(data: Data, fileName: String, mimeType: String) { + Task { await self.addImageAttachment(url: nil, data: data, fileName: fileName, mimeType: mimeType) } + } + + public func removeAttachment(_ id: OpenClawPendingAttachment.ID) { + self.attachments.removeAll { $0.id == id } + } + + public var canSend: Bool { + let trimmed = self.input.trimmingCharacters(in: .whitespacesAndNewlines) + return !self.isSending && self.pendingRunCount == 0 && (!trimmed.isEmpty || !self.attachments.isEmpty) + } + + // MARK: - Internals + + private func bootstrap() async { + self.isLoading = true + self.errorText = nil + self.healthOK = false + self.clearPendingRuns(reason: nil) + self.pendingToolCallsById = [:] + self.streamingAssistantText = nil + self.sessionId = nil + defer { self.isLoading = false } + do { + do { + try await self.transport.setActiveSessionKey(self.sessionKey) + } catch { + // Best-effort only; history/send/health still work without push events. + } + + let payload = try await self.transport.requestHistory(sessionKey: self.sessionKey) + self.messages = Self.decodeMessages(payload.messages ?? []) + self.sessionId = payload.sessionId + if let level = payload.thinkingLevel, !level.isEmpty { + self.thinkingLevel = level + } + await self.pollHealthIfNeeded(force: true) + await self.fetchSessions(limit: 50) + self.errorText = nil + } catch { + self.errorText = error.localizedDescription + chatUILogger.error("bootstrap failed \(error.localizedDescription, privacy: .public)") + } + } + + private static func decodeMessages(_ raw: [AnyCodable]) -> [OpenClawChatMessage] { + let decoded = raw.compactMap { item in + (try? ChatPayloadDecoding.decode(item, as: OpenClawChatMessage.self)) + } + return Self.dedupeMessages(decoded) + } + + private static func dedupeMessages(_ messages: [OpenClawChatMessage]) -> [OpenClawChatMessage] { + var result: [OpenClawChatMessage] = [] + result.reserveCapacity(messages.count) + var seen = Set() + + for message in messages { + guard let key = Self.dedupeKey(for: message) else { + result.append(message) + continue + } + if seen.contains(key) { continue } + seen.insert(key) + result.append(message) + } + + return result + } + + private static func dedupeKey(for message: OpenClawChatMessage) -> String? { + guard let timestamp = message.timestamp else { return nil } + let text = message.content.compactMap(\.text).joined(separator: "\n") + .trimmingCharacters(in: .whitespacesAndNewlines) + guard !text.isEmpty else { return nil } + return "\(message.role)|\(timestamp)|\(text)" + } + + private func performSend() async { + guard !self.isSending else { return } + let trimmed = self.input.trimmingCharacters(in: .whitespacesAndNewlines) + guard !trimmed.isEmpty || !self.attachments.isEmpty else { return } + + guard self.healthOK else { + self.errorText = "Gateway health not OK; cannot send" + return + } + + self.isSending = true + self.errorText = nil + let runId = UUID().uuidString + let messageText = trimmed.isEmpty && !self.attachments.isEmpty ? "See attached." : trimmed + self.pendingRuns.insert(runId) + self.armPendingRunTimeout(runId: runId) + self.pendingToolCallsById = [:] + self.streamingAssistantText = nil + + // Optimistically append user message to UI. + var userContent: [OpenClawChatMessageContent] = [ + OpenClawChatMessageContent( + type: "text", + text: messageText, + thinking: nil, + thinkingSignature: nil, + mimeType: nil, + fileName: nil, + content: nil, + id: nil, + name: nil, + arguments: nil), + ] + let encodedAttachments = self.attachments.map { att -> OpenClawChatAttachmentPayload in + OpenClawChatAttachmentPayload( + type: att.type, + mimeType: att.mimeType, + fileName: att.fileName, + content: att.data.base64EncodedString()) + } + for att in encodedAttachments { + userContent.append( + OpenClawChatMessageContent( + type: att.type, + text: nil, + thinking: nil, + thinkingSignature: nil, + mimeType: att.mimeType, + fileName: att.fileName, + content: AnyCodable(att.content), + id: nil, + name: nil, + arguments: nil)) + } + self.messages.append( + OpenClawChatMessage( + id: UUID(), + role: "user", + content: userContent, + timestamp: Date().timeIntervalSince1970 * 1000)) + + // Clear input immediately for responsive UX (before network await) + self.input = "" + self.attachments = [] + + do { + let response = try await self.transport.sendMessage( + sessionKey: self.sessionKey, + message: messageText, + thinking: self.thinkingLevel, + idempotencyKey: runId, + attachments: encodedAttachments) + if response.runId != runId { + self.clearPendingRun(runId) + self.pendingRuns.insert(response.runId) + self.armPendingRunTimeout(runId: response.runId) + } + } catch { + self.clearPendingRun(runId) + self.errorText = error.localizedDescription + chatUILogger.error("chat.send failed \(error.localizedDescription, privacy: .public)") + } + + self.isSending = false + } + + private func performAbort() async { + guard !self.pendingRuns.isEmpty else { return } + guard !self.isAborting else { return } + self.isAborting = true + defer { self.isAborting = false } + + let runIds = Array(self.pendingRuns) + for runId in runIds { + do { + try await self.transport.abortRun(sessionKey: self.sessionKey, runId: runId) + } catch { + // Best-effort. + } + } + } + + private func fetchSessions(limit: Int?) async { + do { + let res = try await self.transport.listSessions(limit: limit) + self.sessions = res.sessions + } catch { + // Best-effort. + } + } + + private func performSwitchSession(to sessionKey: String) async { + let next = sessionKey.trimmingCharacters(in: .whitespacesAndNewlines) + guard !next.isEmpty else { return } + guard next != self.sessionKey else { return } + self.sessionKey = next + await self.bootstrap() + } + + private func placeholderSession(key: String) -> OpenClawChatSessionEntry { + OpenClawChatSessionEntry( + key: key, + kind: nil, + displayName: nil, + surface: nil, + subject: nil, + room: nil, + space: nil, + updatedAt: nil, + sessionId: nil, + systemSent: nil, + abortedLastRun: nil, + thinkingLevel: nil, + verboseLevel: nil, + inputTokens: nil, + outputTokens: nil, + totalTokens: nil, + model: nil, + contextTokens: nil) + } + + private func handleTransportEvent(_ evt: OpenClawChatTransportEvent) { + switch evt { + case let .health(ok): + self.healthOK = ok + case .tick: + Task { await self.pollHealthIfNeeded(force: false) } + case let .chat(chat): + self.handleChatEvent(chat) + case let .agent(agent): + self.handleAgentEvent(agent) + case .seqGap: + self.errorText = "Event stream interrupted; try refreshing." + self.clearPendingRuns(reason: nil) + } + } + + private func handleChatEvent(_ chat: OpenClawChatEventPayload) { + if let sessionKey = chat.sessionKey, sessionKey != self.sessionKey { + return + } + + let isOurRun = chat.runId.flatMap { self.pendingRuns.contains($0) } ?? false + if !isOurRun { + // Keep multiple clients in sync: if another client finishes a run for our session, refresh history. + switch chat.state { + case "final", "aborted", "error": + self.streamingAssistantText = nil + self.pendingToolCallsById = [:] + Task { await self.refreshHistoryAfterRun() } + default: + break + } + return + } + + switch chat.state { + case "final", "aborted", "error": + if chat.state == "error" { + self.errorText = chat.errorMessage ?? "Chat failed" + } + if let runId = chat.runId { + self.clearPendingRun(runId) + } else if self.pendingRuns.count <= 1 { + self.clearPendingRuns(reason: nil) + } + self.pendingToolCallsById = [:] + self.streamingAssistantText = nil + Task { await self.refreshHistoryAfterRun() } + default: + break + } + } + + private func handleAgentEvent(_ evt: OpenClawAgentEventPayload) { + if let sessionId, evt.runId != sessionId { + return + } + + switch evt.stream { + case "assistant": + if let text = evt.data["text"]?.value as? String { + self.streamingAssistantText = text + } + case "tool": + guard let phase = evt.data["phase"]?.value as? String else { return } + guard let name = evt.data["name"]?.value as? String else { return } + guard let toolCallId = evt.data["toolCallId"]?.value as? String else { return } + if phase == "start" { + let args = evt.data["args"] + self.pendingToolCallsById[toolCallId] = OpenClawChatPendingToolCall( + toolCallId: toolCallId, + name: name, + args: args, + startedAt: evt.ts.map(Double.init) ?? Date().timeIntervalSince1970 * 1000, + isError: nil) + } else if phase == "result" { + self.pendingToolCallsById[toolCallId] = nil + } + default: + break + } + } + + private func refreshHistoryAfterRun() async { + do { + let payload = try await self.transport.requestHistory(sessionKey: self.sessionKey) + self.messages = Self.decodeMessages(payload.messages ?? []) + self.sessionId = payload.sessionId + if let level = payload.thinkingLevel, !level.isEmpty { + self.thinkingLevel = level + } + } catch { + chatUILogger.error("refresh history failed \(error.localizedDescription, privacy: .public)") + } + } + + private func armPendingRunTimeout(runId: String) { + self.pendingRunTimeoutTasks[runId]?.cancel() + self.pendingRunTimeoutTasks[runId] = Task { [weak self] in + let timeoutMs = await MainActor.run { self?.pendingRunTimeoutMs ?? 0 } + try? await Task.sleep(nanoseconds: timeoutMs * 1_000_000) + await MainActor.run { [weak self] in + guard let self else { return } + guard self.pendingRuns.contains(runId) else { return } + self.clearPendingRun(runId) + self.errorText = "Timed out waiting for a reply; try again or refresh." + } + } + } + + private func clearPendingRun(_ runId: String) { + self.pendingRuns.remove(runId) + self.pendingRunTimeoutTasks[runId]?.cancel() + self.pendingRunTimeoutTasks[runId] = nil + } + + private func clearPendingRuns(reason: String?) { + for runId in self.pendingRuns { + self.pendingRunTimeoutTasks[runId]?.cancel() + } + self.pendingRunTimeoutTasks.removeAll() + self.pendingRuns.removeAll() + if let reason, !reason.isEmpty { + self.errorText = reason + } + } + + private func pollHealthIfNeeded(force: Bool) async { + if !force, let last = self.lastHealthPollAt, Date().timeIntervalSince(last) < 10 { + return + } + self.lastHealthPollAt = Date() + do { + let ok = try await self.transport.requestHealth(timeoutMs: 5000) + self.healthOK = ok + } catch { + self.healthOK = false + } + } + + private func loadAttachments(urls: [URL]) async { + for url in urls { + do { + let data = try await Task.detached { try Data(contentsOf: url) }.value + await self.addImageAttachment( + url: url, + data: data, + fileName: url.lastPathComponent, + mimeType: Self.mimeType(for: url) ?? "application/octet-stream") + } catch { + await MainActor.run { self.errorText = error.localizedDescription } + } + } + } + + private static func mimeType(for url: URL) -> String? { + let ext = url.pathExtension + guard !ext.isEmpty else { return nil } + return (UTType(filenameExtension: ext) ?? .data).preferredMIMEType + } + + private func addImageAttachment(url: URL?, data: Data, fileName: String, mimeType: String) async { + if data.count > 5_000_000 { + self.errorText = "Attachment \(fileName) exceeds 5 MB limit" + return + } + + let uti: UTType = { + if let url { + return UTType(filenameExtension: url.pathExtension) ?? .data + } + return UTType(mimeType: mimeType) ?? .data + }() + guard uti.conforms(to: .image) else { + self.errorText = "Only image attachments are supported right now" + return + } + + let preview = Self.previewImage(data: data) + self.attachments.append( + OpenClawPendingAttachment( + url: url, + data: data, + fileName: fileName, + mimeType: mimeType, + preview: preview)) + } + + private static func previewImage(data: Data) -> OpenClawPlatformImage? { + #if canImport(AppKit) + NSImage(data: data) + #elseif canImport(UIKit) + UIImage(data: data) + #else + nil + #endif + } +} diff --git a/apps/shared/OpenClawKit/Sources/OpenClawKit/AnyCodable.swift b/apps/shared/OpenClawKit/Sources/OpenClawKit/AnyCodable.swift new file mode 100644 index 0000000000000000000000000000000000000000..ef522447f43c8f96f1c1d4d5c6d8ea8c5c4f2c38 --- /dev/null +++ b/apps/shared/OpenClawKit/Sources/OpenClawKit/AnyCodable.swift @@ -0,0 +1,93 @@ +import Foundation + +/// Lightweight `Codable` wrapper that round-trips heterogeneous JSON payloads. +/// +/// Marked `@unchecked Sendable` because it can hold reference types. +public struct AnyCodable: Codable, @unchecked Sendable, Hashable { + public let value: Any + + public init(_ value: Any) { self.value = value } + + public init(from decoder: Decoder) throws { + let container = try decoder.singleValueContainer() + if let intVal = try? container.decode(Int.self) { self.value = intVal; return } + if let doubleVal = try? container.decode(Double.self) { self.value = doubleVal; return } + if let boolVal = try? container.decode(Bool.self) { self.value = boolVal; return } + if let stringVal = try? container.decode(String.self) { self.value = stringVal; return } + if container.decodeNil() { self.value = NSNull(); return } + if let dict = try? container.decode([String: AnyCodable].self) { self.value = dict; return } + if let array = try? container.decode([AnyCodable].self) { self.value = array; return } + throw DecodingError.dataCorruptedError(in: container, debugDescription: "Unsupported type") + } + + public func encode(to encoder: Encoder) throws { + var container = encoder.singleValueContainer() + switch self.value { + case let intVal as Int: try container.encode(intVal) + case let doubleVal as Double: try container.encode(doubleVal) + case let boolVal as Bool: try container.encode(boolVal) + case let stringVal as String: try container.encode(stringVal) + case is NSNull: try container.encodeNil() + case let dict as [String: AnyCodable]: try container.encode(dict) + case let array as [AnyCodable]: try container.encode(array) + case let dict as [String: Any]: + try container.encode(dict.mapValues { AnyCodable($0) }) + case let array as [Any]: + try container.encode(array.map { AnyCodable($0) }) + case let dict as NSDictionary: + var converted: [String: AnyCodable] = [:] + for (k, v) in dict { + guard let key = k as? String else { continue } + converted[key] = AnyCodable(v) + } + try container.encode(converted) + case let array as NSArray: + try container.encode(array.map { AnyCodable($0) }) + default: + let context = EncodingError.Context(codingPath: encoder.codingPath, debugDescription: "Unsupported type") + throw EncodingError.invalidValue(self.value, context) + } + } + + public static func == (lhs: AnyCodable, rhs: AnyCodable) -> Bool { + switch (lhs.value, rhs.value) { + case let (l as Int, r as Int): l == r + case let (l as Double, r as Double): l == r + case let (l as Bool, r as Bool): l == r + case let (l as String, r as String): l == r + case (_ as NSNull, _ as NSNull): true + case let (l as [String: AnyCodable], r as [String: AnyCodable]): l == r + case let (l as [AnyCodable], r as [AnyCodable]): l == r + default: + false + } + } + + public func hash(into hasher: inout Hasher) { + switch self.value { + case let v as Int: + hasher.combine(0); hasher.combine(v) + case let v as Double: + hasher.combine(1); hasher.combine(v) + case let v as Bool: + hasher.combine(2); hasher.combine(v) + case let v as String: + hasher.combine(3); hasher.combine(v) + case _ as NSNull: + hasher.combine(4) + case let v as [String: AnyCodable]: + hasher.combine(5) + for (k, val) in v.sorted(by: { $0.key < $1.key }) { + hasher.combine(k) + hasher.combine(val) + } + case let v as [AnyCodable]: + hasher.combine(6) + for item in v { + hasher.combine(item) + } + default: + hasher.combine(999) + } + } +} diff --git a/apps/shared/OpenClawKit/Sources/OpenClawKit/AsyncTimeout.swift b/apps/shared/OpenClawKit/Sources/OpenClawKit/AsyncTimeout.swift new file mode 100644 index 0000000000000000000000000000000000000000..eed2d758ae767c36ed73e05ab8969f710b6082f6 --- /dev/null +++ b/apps/shared/OpenClawKit/Sources/OpenClawKit/AsyncTimeout.swift @@ -0,0 +1,36 @@ +import Foundation + +public enum AsyncTimeout { + public static func withTimeout( + seconds: Double, + onTimeout: @escaping @Sendable () -> Error, + operation: @escaping @Sendable () async throws -> T) async throws -> T + { + let clamped = max(0, seconds) + if clamped == 0 { + return try await operation() + } + + return try await withThrowingTaskGroup(of: T.self) { group in + group.addTask { try await operation() } + group.addTask { + try await Task.sleep(nanoseconds: UInt64(clamped * 1_000_000_000)) + throw onTimeout() + } + let result = try await group.next() + group.cancelAll() + if let result { return result } + throw onTimeout() + } + } + + public static func withTimeoutMs( + timeoutMs: Int, + onTimeout: @escaping @Sendable () -> Error, + operation: @escaping @Sendable () async throws -> T) async throws -> T + { + let clamped = max(0, timeoutMs) + let seconds = Double(clamped) / 1000.0 + return try await self.withTimeout(seconds: seconds, onTimeout: onTimeout, operation: operation) + } +} diff --git a/apps/shared/OpenClawKit/Sources/OpenClawKit/AudioStreamingProtocols.swift b/apps/shared/OpenClawKit/Sources/OpenClawKit/AudioStreamingProtocols.swift new file mode 100644 index 0000000000000000000000000000000000000000..a211a4b3a2ab53425abafae0af4b9cac761f229e --- /dev/null +++ b/apps/shared/OpenClawKit/Sources/OpenClawKit/AudioStreamingProtocols.swift @@ -0,0 +1,16 @@ +import Foundation + +@MainActor +public protocol StreamingAudioPlaying { + func play(stream: AsyncThrowingStream) async -> StreamingPlaybackResult + func stop() -> Double? +} + +@MainActor +public protocol PCMStreamingAudioPlaying { + func play(stream: AsyncThrowingStream, sampleRate: Double) async -> StreamingPlaybackResult + func stop() -> Double? +} + +extension StreamingAudioPlayer: StreamingAudioPlaying {} +extension PCMStreamingAudioPlayer: PCMStreamingAudioPlaying {} diff --git a/apps/shared/OpenClawKit/Sources/OpenClawKit/BonjourEscapes.swift b/apps/shared/OpenClawKit/Sources/OpenClawKit/BonjourEscapes.swift new file mode 100644 index 0000000000000000000000000000000000000000..0760314f72702e9d5e869beaf2a2f566d80af4a7 --- /dev/null +++ b/apps/shared/OpenClawKit/Sources/OpenClawKit/BonjourEscapes.swift @@ -0,0 +1,33 @@ +import Foundation + +public enum BonjourEscapes { + /// mDNS / DNS-SD commonly escapes bytes in instance names as `\DDD` (decimal-encoded), + /// e.g. spaces are `\032`. + public static func decode(_ input: String) -> String { + var out = "" + var i = input.startIndex + while i < input.endIndex { + if input[i] == "\\", + let d0 = input.index(i, offsetBy: 1, limitedBy: input.index(before: input.endIndex)), + let d1 = input.index(i, offsetBy: 2, limitedBy: input.index(before: input.endIndex)), + let d2 = input.index(i, offsetBy: 3, limitedBy: input.index(before: input.endIndex)), + input[d0].isNumber, + input[d1].isNumber, + input[d2].isNumber + { + let digits = String(input[d0...d2]) + if let value = Int(digits), + let scalar = UnicodeScalar(value) + { + out.append(Character(scalar)) + i = input.index(i, offsetBy: 4) + continue + } + } + + out.append(input[i]) + i = input.index(after: i) + } + return out + } +} diff --git a/apps/shared/OpenClawKit/Sources/OpenClawKit/BonjourTypes.swift b/apps/shared/OpenClawKit/Sources/OpenClawKit/BonjourTypes.swift new file mode 100644 index 0000000000000000000000000000000000000000..5c3c50ca482f97f2cf307f6b4b0c59621fb12bfe --- /dev/null +++ b/apps/shared/OpenClawKit/Sources/OpenClawKit/BonjourTypes.swift @@ -0,0 +1,40 @@ +import Foundation + +public enum OpenClawBonjour { + // v0: internal-only, subject to rename. + public static let gatewayServiceType = "_openclaw-gw._tcp" + public static let gatewayServiceDomain = "local." + public static var wideAreaGatewayServiceDomain: String? { + let env = ProcessInfo.processInfo.environment + return resolveWideAreaDomain(env["OPENCLAW_WIDE_AREA_DOMAIN"]) + } + + public static var gatewayServiceDomains: [String] { + var domains = [gatewayServiceDomain] + if let wideArea = wideAreaGatewayServiceDomain { + domains.append(wideArea) + } + return domains + } + + private static func resolveWideAreaDomain(_ raw: String?) -> String? { + let trimmed = (raw ?? "").trimmingCharacters(in: .whitespacesAndNewlines) + if trimmed.isEmpty { return nil } + let normalized = normalizeServiceDomain(trimmed) + return normalized == gatewayServiceDomain ? nil : normalized + } + + public static func normalizeServiceDomain(_ raw: String?) -> String { + let trimmed = (raw ?? "").trimmingCharacters(in: .whitespacesAndNewlines) + if trimmed.isEmpty { + return self.gatewayServiceDomain + } + + let lower = trimmed.lowercased() + if lower == "local" || lower == "local." { + return self.gatewayServiceDomain + } + + return lower.hasSuffix(".") ? lower : (lower + ".") + } +} diff --git a/apps/shared/OpenClawKit/Sources/OpenClawKit/BridgeFrames.swift b/apps/shared/OpenClawKit/Sources/OpenClawKit/BridgeFrames.swift new file mode 100644 index 0000000000000000000000000000000000000000..648b257bbb497cb5f749c3b9cb9c94a462c810a0 --- /dev/null +++ b/apps/shared/OpenClawKit/Sources/OpenClawKit/BridgeFrames.swift @@ -0,0 +1,261 @@ +import Foundation + +public struct BridgeBaseFrame: Codable, Sendable { + public let type: String + + public init(type: String) { + self.type = type + } +} + +public struct BridgeInvokeRequest: Codable, Sendable { + public let type: String + public let id: String + public let command: String + public let paramsJSON: String? + + public init(type: String = "invoke", id: String, command: String, paramsJSON: String? = nil) { + self.type = type + self.id = id + self.command = command + self.paramsJSON = paramsJSON + } +} + +public struct BridgeInvokeResponse: Codable, Sendable { + public let type: String + public let id: String + public let ok: Bool + public let payloadJSON: String? + public let error: OpenClawNodeError? + + public init( + type: String = "invoke-res", + id: String, + ok: Bool, + payloadJSON: String? = nil, + error: OpenClawNodeError? = nil) + { + self.type = type + self.id = id + self.ok = ok + self.payloadJSON = payloadJSON + self.error = error + } +} + +public struct BridgeEventFrame: Codable, Sendable { + public let type: String + public let event: String + public let payloadJSON: String? + + public init(type: String = "event", event: String, payloadJSON: String? = nil) { + self.type = type + self.event = event + self.payloadJSON = payloadJSON + } +} + +public struct BridgeHello: Codable, Sendable { + public let type: String + public let nodeId: String + public let displayName: String? + public let token: String? + public let platform: String? + public let version: String? + public let coreVersion: String? + public let uiVersion: String? + public let deviceFamily: String? + public let modelIdentifier: String? + public let caps: [String]? + public let commands: [String]? + public let permissions: [String: Bool]? + + public init( + type: String = "hello", + nodeId: String, + displayName: String?, + token: String?, + platform: String?, + version: String?, + coreVersion: String? = nil, + uiVersion: String? = nil, + deviceFamily: String? = nil, + modelIdentifier: String? = nil, + caps: [String]? = nil, + commands: [String]? = nil, + permissions: [String: Bool]? = nil) + { + self.type = type + self.nodeId = nodeId + self.displayName = displayName + self.token = token + self.platform = platform + self.version = version + self.coreVersion = coreVersion + self.uiVersion = uiVersion + self.deviceFamily = deviceFamily + self.modelIdentifier = modelIdentifier + self.caps = caps + self.commands = commands + self.permissions = permissions + } +} + +public struct BridgeHelloOk: Codable, Sendable { + public let type: String + public let serverName: String + public let canvasHostUrl: String? + public let mainSessionKey: String? + + public init( + type: String = "hello-ok", + serverName: String, + canvasHostUrl: String? = nil, + mainSessionKey: String? = nil) + { + self.type = type + self.serverName = serverName + self.canvasHostUrl = canvasHostUrl + self.mainSessionKey = mainSessionKey + } +} + +public struct BridgePairRequest: Codable, Sendable { + public let type: String + public let nodeId: String + public let displayName: String? + public let platform: String? + public let version: String? + public let coreVersion: String? + public let uiVersion: String? + public let deviceFamily: String? + public let modelIdentifier: String? + public let caps: [String]? + public let commands: [String]? + public let permissions: [String: Bool]? + public let remoteAddress: String? + public let silent: Bool? + + public init( + type: String = "pair-request", + nodeId: String, + displayName: String?, + platform: String?, + version: String?, + coreVersion: String? = nil, + uiVersion: String? = nil, + deviceFamily: String? = nil, + modelIdentifier: String? = nil, + caps: [String]? = nil, + commands: [String]? = nil, + permissions: [String: Bool]? = nil, + remoteAddress: String? = nil, + silent: Bool? = nil) + { + self.type = type + self.nodeId = nodeId + self.displayName = displayName + self.platform = platform + self.version = version + self.coreVersion = coreVersion + self.uiVersion = uiVersion + self.deviceFamily = deviceFamily + self.modelIdentifier = modelIdentifier + self.caps = caps + self.commands = commands + self.permissions = permissions + self.remoteAddress = remoteAddress + self.silent = silent + } +} + +public struct BridgePairOk: Codable, Sendable { + public let type: String + public let token: String + + public init(type: String = "pair-ok", token: String) { + self.type = type + self.token = token + } +} + +public struct BridgePing: Codable, Sendable { + public let type: String + public let id: String + + public init(type: String = "ping", id: String) { + self.type = type + self.id = id + } +} + +public struct BridgePong: Codable, Sendable { + public let type: String + public let id: String + + public init(type: String = "pong", id: String) { + self.type = type + self.id = id + } +} + +public struct BridgeErrorFrame: Codable, Sendable { + public let type: String + public let code: String + public let message: String + + public init(type: String = "error", code: String, message: String) { + self.type = type + self.code = code + self.message = message + } +} + +// MARK: - Optional RPC (node -> bridge) + +public struct BridgeRPCRequest: Codable, Sendable { + public let type: String + public let id: String + public let method: String + public let paramsJSON: String? + + public init(type: String = "req", id: String, method: String, paramsJSON: String? = nil) { + self.type = type + self.id = id + self.method = method + self.paramsJSON = paramsJSON + } +} + +public struct BridgeRPCError: Codable, Sendable, Equatable { + public let code: String + public let message: String + + public init(code: String, message: String) { + self.code = code + self.message = message + } +} + +public struct BridgeRPCResponse: Codable, Sendable { + public let type: String + public let id: String + public let ok: Bool + public let payloadJSON: String? + public let error: BridgeRPCError? + + public init( + type: String = "res", + id: String, + ok: Bool, + payloadJSON: String? = nil, + error: BridgeRPCError? = nil) + { + self.type = type + self.id = id + self.ok = ok + self.payloadJSON = payloadJSON + self.error = error + } +} diff --git a/apps/shared/OpenClawKit/Sources/OpenClawKit/CameraCommands.swift b/apps/shared/OpenClawKit/Sources/OpenClawKit/CameraCommands.swift new file mode 100644 index 0000000000000000000000000000000000000000..c76ff8e97f9418988868327d7ba4dc3fc4fa7ac8 --- /dev/null +++ b/apps/shared/OpenClawKit/Sources/OpenClawKit/CameraCommands.swift @@ -0,0 +1,68 @@ +import Foundation + +public enum OpenClawCameraCommand: String, Codable, Sendable { + case list = "camera.list" + case snap = "camera.snap" + case clip = "camera.clip" +} + +public enum OpenClawCameraFacing: String, Codable, Sendable { + case back + case front +} + +public enum OpenClawCameraImageFormat: String, Codable, Sendable { + case jpg + case jpeg +} + +public enum OpenClawCameraVideoFormat: String, Codable, Sendable { + case mp4 +} + +public struct OpenClawCameraSnapParams: Codable, Sendable, Equatable { + public var facing: OpenClawCameraFacing? + public var maxWidth: Int? + public var quality: Double? + public var format: OpenClawCameraImageFormat? + public var deviceId: String? + public var delayMs: Int? + + public init( + facing: OpenClawCameraFacing? = nil, + maxWidth: Int? = nil, + quality: Double? = nil, + format: OpenClawCameraImageFormat? = nil, + deviceId: String? = nil, + delayMs: Int? = nil) + { + self.facing = facing + self.maxWidth = maxWidth + self.quality = quality + self.format = format + self.deviceId = deviceId + self.delayMs = delayMs + } +} + +public struct OpenClawCameraClipParams: Codable, Sendable, Equatable { + public var facing: OpenClawCameraFacing? + public var durationMs: Int? + public var includeAudio: Bool? + public var format: OpenClawCameraVideoFormat? + public var deviceId: String? + + public init( + facing: OpenClawCameraFacing? = nil, + durationMs: Int? = nil, + includeAudio: Bool? = nil, + format: OpenClawCameraVideoFormat? = nil, + deviceId: String? = nil) + { + self.facing = facing + self.durationMs = durationMs + self.includeAudio = includeAudio + self.format = format + self.deviceId = deviceId + } +} diff --git a/apps/shared/OpenClawKit/Sources/OpenClawKit/CanvasA2UIAction.swift b/apps/shared/OpenClawKit/Sources/OpenClawKit/CanvasA2UIAction.swift new file mode 100644 index 0000000000000000000000000000000000000000..909f89a441f8bf5ebeb1875f8f282a2249fa1f87 --- /dev/null +++ b/apps/shared/OpenClawKit/Sources/OpenClawKit/CanvasA2UIAction.swift @@ -0,0 +1,104 @@ +import Foundation + +public enum OpenClawCanvasA2UIAction: Sendable { + public struct AgentMessageContext: Sendable { + public struct Session: Sendable { + public var key: String + public var surfaceId: String + + public init(key: String, surfaceId: String) { + self.key = key + self.surfaceId = surfaceId + } + } + + public struct Component: Sendable { + public var id: String + public var host: String + public var instanceId: String + + public init(id: String, host: String, instanceId: String) { + self.id = id + self.host = host + self.instanceId = instanceId + } + } + + public var actionName: String + public var session: Session + public var component: Component + public var contextJSON: String? + + public init(actionName: String, session: Session, component: Component, contextJSON: String?) { + self.actionName = actionName + self.session = session + self.component = component + self.contextJSON = contextJSON + } + } + + public static func extractActionName(_ userAction: [String: Any]) -> String? { + let keys = ["name", "action"] + for key in keys { + if let raw = userAction[key] as? String { + let trimmed = raw.trimmingCharacters(in: .whitespacesAndNewlines) + if !trimmed.isEmpty { return trimmed } + } + } + return nil + } + + public static func sanitizeTagValue(_ value: String) -> String { + let trimmed = value.trimmingCharacters(in: .whitespacesAndNewlines) + let nonEmpty = trimmed.isEmpty ? "-" : trimmed + let normalized = nonEmpty.replacingOccurrences(of: " ", with: "_") + let allowed = CharacterSet(charactersIn: "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_-.:") + let scalars = normalized.unicodeScalars.map { allowed.contains($0) ? Character($0) : "_" } + return String(scalars) + } + + public static func compactJSON(_ obj: Any?) -> String? { + guard let obj else { return nil } + guard JSONSerialization.isValidJSONObject(obj) else { return nil } + guard let data = try? JSONSerialization.data(withJSONObject: obj, options: []), + let str = String(data: data, encoding: .utf8) + else { return nil } + return str + } + + public static func formatAgentMessage(_ context: AgentMessageContext) -> String { + let ctxSuffix = context.contextJSON.flatMap { $0.isEmpty ? nil : " ctx=\($0)" } ?? "" + return [ + "CANVAS_A2UI", + "action=\(self.sanitizeTagValue(context.actionName))", + "session=\(self.sanitizeTagValue(context.session.key))", + "surface=\(self.sanitizeTagValue(context.session.surfaceId))", + "component=\(self.sanitizeTagValue(context.component.id))", + "host=\(self.sanitizeTagValue(context.component.host))", + "instance=\(self.sanitizeTagValue(context.component.instanceId))\(ctxSuffix)", + "default=update_canvas", + ].joined(separator: " ") + } + + public static func jsDispatchA2UIActionStatus(actionId: String, ok: Bool, error: String?) -> String { + let payload: [String: Any] = [ + "id": actionId, + "ok": ok, + "error": error ?? "", + ] + let json: String = { + if let data = try? JSONSerialization.data(withJSONObject: payload, options: []), + let str = String(data: data, encoding: .utf8) + { + return str + } + return "{\"id\":\"\(actionId)\",\"ok\":\(ok ? "true" : "false"),\"error\":\"\"}" + }() + return """ + (() => { + const detail = \(json); + window.dispatchEvent(new CustomEvent('openclaw:a2ui-action-status', { detail })); + })(); + """ + } +} diff --git a/apps/shared/OpenClawKit/Sources/OpenClawKit/CanvasA2UICommands.swift b/apps/shared/OpenClawKit/Sources/OpenClawKit/CanvasA2UICommands.swift new file mode 100644 index 0000000000000000000000000000000000000000..ab3af0c367a515e04737055eb00165683b931bd0 --- /dev/null +++ b/apps/shared/OpenClawKit/Sources/OpenClawKit/CanvasA2UICommands.swift @@ -0,0 +1,26 @@ +import Foundation + +public enum OpenClawCanvasA2UICommand: String, Codable, Sendable { + /// Render A2UI content on the device canvas. + case push = "canvas.a2ui.push" + /// Legacy alias for `push` when sending JSONL. + case pushJSONL = "canvas.a2ui.pushJSONL" + /// Reset the A2UI renderer state. + case reset = "canvas.a2ui.reset" +} + +public struct OpenClawCanvasA2UIPushParams: Codable, Sendable, Equatable { + public var messages: [AnyCodable] + + public init(messages: [AnyCodable]) { + self.messages = messages + } +} + +public struct OpenClawCanvasA2UIPushJSONLParams: Codable, Sendable, Equatable { + public var jsonl: String + + public init(jsonl: String) { + self.jsonl = jsonl + } +} diff --git a/apps/shared/OpenClawKit/Sources/OpenClawKit/CanvasA2UIJSONL.swift b/apps/shared/OpenClawKit/Sources/OpenClawKit/CanvasA2UIJSONL.swift new file mode 100644 index 0000000000000000000000000000000000000000..d5026a8be7bf7d4c4f49ca465dd301f93cfb5891 --- /dev/null +++ b/apps/shared/OpenClawKit/Sources/OpenClawKit/CanvasA2UIJSONL.swift @@ -0,0 +1,81 @@ +import Foundation + +public enum OpenClawCanvasA2UIJSONL: Sendable { + public struct ParsedItem: Sendable { + public var lineNumber: Int + public var message: AnyCodable + + public init(lineNumber: Int, message: AnyCodable) { + self.lineNumber = lineNumber + self.message = message + } + } + + public static func parse(_ text: String) throws -> [ParsedItem] { + var out: [ParsedItem] = [] + var lineNumber = 0 + for rawLine in text.split(omittingEmptySubsequences: false, whereSeparator: \.isNewline) { + lineNumber += 1 + let line = String(rawLine).trimmingCharacters(in: .whitespacesAndNewlines) + if line.isEmpty { continue } + let data = Data(line.utf8) + + let decoded = try JSONDecoder().decode(AnyCodable.self, from: data) + out.append(ParsedItem(lineNumber: lineNumber, message: decoded)) + } + return out + } + + public static func validateV0_8(_ items: [ParsedItem]) throws { + let allowed = Set([ + "beginRendering", + "surfaceUpdate", + "dataModelUpdate", + "deleteSurface", + ]) + for item in items { + guard let dict = item.message.value as? [String: AnyCodable] else { + throw NSError(domain: "A2UI", code: 1, userInfo: [ + NSLocalizedDescriptionKey: "A2UI JSONL line \(item.lineNumber): expected a JSON object", + ]) + } + + if dict.keys.contains("createSurface") { + throw NSError(domain: "A2UI", code: 2, userInfo: [ + NSLocalizedDescriptionKey: """ + A2UI JSONL line \(item.lineNumber): looks like A2UI v0.9 (`createSurface`). + Canvas currently supports A2UI v0.8 server→client messages + (`beginRendering`, `surfaceUpdate`, `dataModelUpdate`, `deleteSurface`). + """, + ]) + } + + let matched = dict.keys.filter { allowed.contains($0) } + if matched.count != 1 { + let found = dict.keys.sorted().joined(separator: ", ") + throw NSError(domain: "A2UI", code: 3, userInfo: [ + NSLocalizedDescriptionKey: """ + A2UI JSONL line \(item.lineNumber): expected exactly one of \(allowed.sorted() + .joined(separator: ", ")); found: \(found) + """, + ]) + } + } + } + + public static func decodeMessagesFromJSONL(_ text: String) throws -> [AnyCodable] { + let items = try self.parse(text) + try self.validateV0_8(items) + return items.map(\.message) + } + + public static func encodeMessagesJSONArray(_ messages: [AnyCodable]) throws -> String { + let data = try JSONEncoder().encode(messages) + guard let json = String(data: data, encoding: .utf8) else { + throw NSError(domain: "A2UI", code: 10, userInfo: [ + NSLocalizedDescriptionKey: "Failed to encode messages payload as UTF-8", + ]) + } + return json + } +} diff --git a/apps/shared/OpenClawKit/Sources/OpenClawKit/CanvasCommandParams.swift b/apps/shared/OpenClawKit/Sources/OpenClawKit/CanvasCommandParams.swift new file mode 100644 index 0000000000000000000000000000000000000000..2c109cf2fdaca8252b4445a67838367ca5bd840e --- /dev/null +++ b/apps/shared/OpenClawKit/Sources/OpenClawKit/CanvasCommandParams.swift @@ -0,0 +1,76 @@ +import Foundation + +public struct OpenClawCanvasNavigateParams: Codable, Sendable, Equatable { + public var url: String + + public init(url: String) { + self.url = url + } +} + +public struct OpenClawCanvasPlacement: Codable, Sendable, Equatable { + public var x: Double? + public var y: Double? + public var width: Double? + public var height: Double? + + public init(x: Double? = nil, y: Double? = nil, width: Double? = nil, height: Double? = nil) { + self.x = x + self.y = y + self.width = width + self.height = height + } +} + +public struct OpenClawCanvasPresentParams: Codable, Sendable, Equatable { + public var url: String? + public var placement: OpenClawCanvasPlacement? + + public init(url: String? = nil, placement: OpenClawCanvasPlacement? = nil) { + self.url = url + self.placement = placement + } +} + +public struct OpenClawCanvasEvalParams: Codable, Sendable, Equatable { + public var javaScript: String + + public init(javaScript: String) { + self.javaScript = javaScript + } +} + +public enum OpenClawCanvasSnapshotFormat: String, Codable, Sendable { + case png + case jpeg + + public init(from decoder: Decoder) throws { + let c = try decoder.singleValueContainer() + let raw = try c.decode(String.self).trimmingCharacters(in: .whitespacesAndNewlines).lowercased() + switch raw { + case "png": + self = .png + case "jpeg", "jpg": + self = .jpeg + default: + throw DecodingError.dataCorruptedError(in: c, debugDescription: "Invalid snapshot format: \(raw)") + } + } + + public func encode(to encoder: Encoder) throws { + var c = encoder.singleValueContainer() + try c.encode(self.rawValue) + } +} + +public struct OpenClawCanvasSnapshotParams: Codable, Sendable, Equatable { + public var maxWidth: Int? + public var quality: Double? + public var format: OpenClawCanvasSnapshotFormat? + + public init(maxWidth: Int? = nil, quality: Double? = nil, format: OpenClawCanvasSnapshotFormat? = nil) { + self.maxWidth = maxWidth + self.quality = quality + self.format = format + } +} diff --git a/apps/shared/OpenClawKit/Sources/OpenClawKit/CanvasCommands.swift b/apps/shared/OpenClawKit/Sources/OpenClawKit/CanvasCommands.swift new file mode 100644 index 0000000000000000000000000000000000000000..544353bc063f72d5bdf8189cafdd7c1f02f267ee --- /dev/null +++ b/apps/shared/OpenClawKit/Sources/OpenClawKit/CanvasCommands.swift @@ -0,0 +1,9 @@ +import Foundation + +public enum OpenClawCanvasCommand: String, Codable, Sendable { + case present = "canvas.present" + case hide = "canvas.hide" + case navigate = "canvas.navigate" + case evalJS = "canvas.eval" + case snapshot = "canvas.snapshot" +} diff --git a/apps/shared/OpenClawKit/Sources/OpenClawKit/Capabilities.swift b/apps/shared/OpenClawKit/Sources/OpenClawKit/Capabilities.swift new file mode 100644 index 0000000000000000000000000000000000000000..1cb820e732227ca828357e91eb79d7c8a4792de8 --- /dev/null +++ b/apps/shared/OpenClawKit/Sources/OpenClawKit/Capabilities.swift @@ -0,0 +1,9 @@ +import Foundation + +public enum OpenClawCapability: String, Codable, Sendable { + case canvas + case camera + case screen + case voiceWake + case location +} diff --git a/apps/shared/OpenClawKit/Sources/OpenClawKit/DeepLinks.swift b/apps/shared/OpenClawKit/Sources/OpenClawKit/DeepLinks.swift new file mode 100644 index 0000000000000000000000000000000000000000..10dd7ea05368ed022beee0471a5ccd3efdac2713 --- /dev/null +++ b/apps/shared/OpenClawKit/Sources/OpenClawKit/DeepLinks.swift @@ -0,0 +1,76 @@ +import Foundation + +public enum DeepLinkRoute: Sendable, Equatable { + case agent(AgentDeepLink) +} + +public struct AgentDeepLink: Codable, Sendable, Equatable { + public let message: String + public let sessionKey: String? + public let thinking: String? + public let deliver: Bool + public let to: String? + public let channel: String? + public let timeoutSeconds: Int? + public let key: String? + + public init( + message: String, + sessionKey: String?, + thinking: String?, + deliver: Bool, + to: String?, + channel: String?, + timeoutSeconds: Int?, + key: String?) + { + self.message = message + self.sessionKey = sessionKey + self.thinking = thinking + self.deliver = deliver + self.to = to + self.channel = channel + self.timeoutSeconds = timeoutSeconds + self.key = key + } +} + +public enum DeepLinkParser { + public static func parse(_ url: URL) -> DeepLinkRoute? { + guard let scheme = url.scheme?.lowercased(), + scheme == "openclaw" + else { + return nil + } + guard let host = url.host?.lowercased(), !host.isEmpty else { return nil } + guard let comps = URLComponents(url: url, resolvingAgainstBaseURL: false) else { return nil } + + let query = (comps.queryItems ?? []).reduce(into: [String: String]()) { dict, item in + guard let value = item.value else { return } + dict[item.name] = value + } + + switch host { + case "agent": + guard let message = query["message"], + !message.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty + else { + return nil + } + let deliver = (query["deliver"] as NSString?)?.boolValue ?? false + let timeoutSeconds = query["timeoutSeconds"].flatMap { Int($0) }.flatMap { $0 >= 0 ? $0 : nil } + return .agent( + .init( + message: message, + sessionKey: query["sessionKey"], + thinking: query["thinking"], + deliver: deliver, + to: query["to"], + channel: query["channel"], + timeoutSeconds: timeoutSeconds, + key: query["key"])) + default: + return nil + } + } +} diff --git a/apps/shared/OpenClawKit/Sources/OpenClawKit/DeviceAuthStore.swift b/apps/shared/OpenClawKit/Sources/OpenClawKit/DeviceAuthStore.swift new file mode 100644 index 0000000000000000000000000000000000000000..80ff20c3f35ad7440641466d71226bab2a98d05b --- /dev/null +++ b/apps/shared/OpenClawKit/Sources/OpenClawKit/DeviceAuthStore.swift @@ -0,0 +1,107 @@ +import Foundation + +public struct DeviceAuthEntry: Codable, Sendable { + public let token: String + public let role: String + public let scopes: [String] + public let updatedAtMs: Int + + public init(token: String, role: String, scopes: [String], updatedAtMs: Int) { + self.token = token + self.role = role + self.scopes = scopes + self.updatedAtMs = updatedAtMs + } +} + +private struct DeviceAuthStoreFile: Codable { + var version: Int + var deviceId: String + var tokens: [String: DeviceAuthEntry] +} + +public enum DeviceAuthStore { + private static let fileName = "device-auth.json" + + public static func loadToken(deviceId: String, role: String) -> DeviceAuthEntry? { + guard let store = readStore(), store.deviceId == deviceId else { return nil } + let role = normalizeRole(role) + return store.tokens[role] + } + + public static func storeToken( + deviceId: String, + role: String, + token: String, + scopes: [String] = [] + ) -> DeviceAuthEntry { + let normalizedRole = normalizeRole(role) + var next = readStore() + if next?.deviceId != deviceId { + next = DeviceAuthStoreFile(version: 1, deviceId: deviceId, tokens: [:]) + } + let entry = DeviceAuthEntry( + token: token, + role: normalizedRole, + scopes: normalizeScopes(scopes), + updatedAtMs: Int(Date().timeIntervalSince1970 * 1000) + ) + if next == nil { + next = DeviceAuthStoreFile(version: 1, deviceId: deviceId, tokens: [:]) + } + next?.tokens[normalizedRole] = entry + if let store = next { + writeStore(store) + } + return entry + } + + public static func clearToken(deviceId: String, role: String) { + guard var store = readStore(), store.deviceId == deviceId else { return } + let normalizedRole = normalizeRole(role) + guard store.tokens[normalizedRole] != nil else { return } + store.tokens.removeValue(forKey: normalizedRole) + writeStore(store) + } + + private static func normalizeRole(_ role: String) -> String { + role.trimmingCharacters(in: .whitespacesAndNewlines) + } + + private static func normalizeScopes(_ scopes: [String]) -> [String] { + let trimmed = scopes + .map { $0.trimmingCharacters(in: .whitespacesAndNewlines) } + .filter { !$0.isEmpty } + return Array(Set(trimmed)).sorted() + } + + private static func fileURL() -> URL { + DeviceIdentityPaths.stateDirURL() + .appendingPathComponent("identity", isDirectory: true) + .appendingPathComponent(fileName, isDirectory: false) + } + + private static func readStore() -> DeviceAuthStoreFile? { + let url = fileURL() + guard let data = try? Data(contentsOf: url) else { return nil } + guard let decoded = try? JSONDecoder().decode(DeviceAuthStoreFile.self, from: data) else { + return nil + } + guard decoded.version == 1 else { return nil } + return decoded + } + + private static func writeStore(_ store: DeviceAuthStoreFile) { + let url = fileURL() + do { + try FileManager.default.createDirectory( + at: url.deletingLastPathComponent(), + withIntermediateDirectories: true) + let data = try JSONEncoder().encode(store) + try data.write(to: url, options: [.atomic]) + try? FileManager.default.setAttributes([.posixPermissions: 0o600], ofItemAtPath: url.path) + } catch { + // best-effort only + } + } +} diff --git a/apps/shared/OpenClawKit/Sources/OpenClawKit/DeviceIdentity.swift b/apps/shared/OpenClawKit/Sources/OpenClawKit/DeviceIdentity.swift new file mode 100644 index 0000000000000000000000000000000000000000..a992bc58f29c2dc2ae14740aeb694c4355da0f3d --- /dev/null +++ b/apps/shared/OpenClawKit/Sources/OpenClawKit/DeviceIdentity.swift @@ -0,0 +1,112 @@ +import CryptoKit +import Foundation + +public struct DeviceIdentity: Codable, Sendable { + public var deviceId: String + public var publicKey: String + public var privateKey: String + public var createdAtMs: Int + + public init(deviceId: String, publicKey: String, privateKey: String, createdAtMs: Int) { + self.deviceId = deviceId + self.publicKey = publicKey + self.privateKey = privateKey + self.createdAtMs = createdAtMs + } +} + +enum DeviceIdentityPaths { + private static let stateDirEnv = ["OPENCLAW_STATE_DIR"] + + static func stateDirURL() -> URL { + for key in self.stateDirEnv { + if let raw = getenv(key) { + let value = String(cString: raw).trimmingCharacters(in: .whitespacesAndNewlines) + if !value.isEmpty { + return URL(fileURLWithPath: value, isDirectory: true) + } + } + } + + if let appSupport = FileManager.default.urls(for: .applicationSupportDirectory, in: .userDomainMask).first { + return appSupport.appendingPathComponent("OpenClaw", isDirectory: true) + } + + return FileManager.default.temporaryDirectory.appendingPathComponent("openclaw", isDirectory: true) + } +} + +public enum DeviceIdentityStore { + private static let fileName = "device.json" + + public static func loadOrCreate() -> DeviceIdentity { + let url = self.fileURL() + if let data = try? Data(contentsOf: url), + let decoded = try? JSONDecoder().decode(DeviceIdentity.self, from: data), + !decoded.deviceId.isEmpty, + !decoded.publicKey.isEmpty, + !decoded.privateKey.isEmpty { + return decoded + } + let identity = self.generate() + self.save(identity) + return identity + } + + public static func signPayload(_ payload: String, identity: DeviceIdentity) -> String? { + guard let privateKeyData = Data(base64Encoded: identity.privateKey) else { return nil } + do { + let privateKey = try Curve25519.Signing.PrivateKey(rawRepresentation: privateKeyData) + let signature = try privateKey.signature(for: Data(payload.utf8)) + return self.base64UrlEncode(signature) + } catch { + return nil + } + } + + private static func generate() -> DeviceIdentity { + let privateKey = Curve25519.Signing.PrivateKey() + let publicKey = privateKey.publicKey + let publicKeyData = publicKey.rawRepresentation + let privateKeyData = privateKey.rawRepresentation + let deviceId = SHA256.hash(data: publicKeyData).compactMap { String(format: "%02x", $0) }.joined() + return DeviceIdentity( + deviceId: deviceId, + publicKey: publicKeyData.base64EncodedString(), + privateKey: privateKeyData.base64EncodedString(), + createdAtMs: Int(Date().timeIntervalSince1970 * 1000)) + } + + private static func base64UrlEncode(_ data: Data) -> String { + let base64 = data.base64EncodedString() + return base64 + .replacingOccurrences(of: "+", with: "-") + .replacingOccurrences(of: "/", with: "_") + .replacingOccurrences(of: "=", with: "") + } + + public static func publicKeyBase64Url(_ identity: DeviceIdentity) -> String? { + guard let data = Data(base64Encoded: identity.publicKey) else { return nil } + return self.base64UrlEncode(data) + } + + private static func save(_ identity: DeviceIdentity) { + let url = self.fileURL() + do { + try FileManager.default.createDirectory( + at: url.deletingLastPathComponent(), + withIntermediateDirectories: true) + let data = try JSONEncoder().encode(identity) + try data.write(to: url, options: [.atomic]) + } catch { + // best-effort only + } + } + + private static func fileURL() -> URL { + let base = DeviceIdentityPaths.stateDirURL() + return base + .appendingPathComponent("identity", isDirectory: true) + .appendingPathComponent(fileName, isDirectory: false) + } +} diff --git a/apps/shared/OpenClawKit/Sources/OpenClawKit/ElevenLabsKitShim.swift b/apps/shared/OpenClawKit/Sources/OpenClawKit/ElevenLabsKitShim.swift new file mode 100644 index 0000000000000000000000000000000000000000..07fe91ac37c1a1e2548a843b0177f18061638566 --- /dev/null +++ b/apps/shared/OpenClawKit/Sources/OpenClawKit/ElevenLabsKitShim.swift @@ -0,0 +1,9 @@ +@_exported import ElevenLabsKit + +public typealias ElevenLabsVoice = ElevenLabsKit.ElevenLabsVoice +public typealias ElevenLabsTTSRequest = ElevenLabsKit.ElevenLabsTTSRequest +public typealias ElevenLabsTTSClient = ElevenLabsKit.ElevenLabsTTSClient +public typealias TalkTTSValidation = ElevenLabsKit.TalkTTSValidation +public typealias StreamingAudioPlayer = ElevenLabsKit.StreamingAudioPlayer +public typealias PCMStreamingAudioPlayer = ElevenLabsKit.PCMStreamingAudioPlayer +public typealias StreamingPlaybackResult = ElevenLabsKit.StreamingPlaybackResult diff --git a/apps/shared/OpenClawKit/Sources/OpenClawKit/GatewayChannel.swift b/apps/shared/OpenClawKit/Sources/OpenClawKit/GatewayChannel.swift new file mode 100644 index 0000000000000000000000000000000000000000..aebfcd72c1190d0a9b26665d91c1e8e8d7a3d97f --- /dev/null +++ b/apps/shared/OpenClawKit/Sources/OpenClawKit/GatewayChannel.swift @@ -0,0 +1,713 @@ +import OpenClawProtocol +import Foundation +import OSLog + +public protocol WebSocketTasking: AnyObject { + var state: URLSessionTask.State { get } + func resume() + func cancel(with closeCode: URLSessionWebSocketTask.CloseCode, reason: Data?) + func send(_ message: URLSessionWebSocketTask.Message) async throws + func receive() async throws -> URLSessionWebSocketTask.Message + func receive(completionHandler: @escaping @Sendable (Result) -> Void) +} + +extension URLSessionWebSocketTask: WebSocketTasking {} + +public struct WebSocketTaskBox: @unchecked Sendable { + public let task: any WebSocketTasking + public init(task: any WebSocketTasking) { + self.task = task + } + + public var state: URLSessionTask.State { self.task.state } + + public func resume() { self.task.resume() } + + public func cancel(with closeCode: URLSessionWebSocketTask.CloseCode, reason: Data?) { + self.task.cancel(with: closeCode, reason: reason) + } + + public func send(_ message: URLSessionWebSocketTask.Message) async throws { + try await self.task.send(message) + } + + public func receive() async throws -> URLSessionWebSocketTask.Message { + try await self.task.receive() + } + + public func receive( + completionHandler: @escaping @Sendable (Result) -> Void) + { + self.task.receive(completionHandler: completionHandler) + } +} + +public protocol WebSocketSessioning: AnyObject { + func makeWebSocketTask(url: URL) -> WebSocketTaskBox +} + +extension URLSession: WebSocketSessioning { + public func makeWebSocketTask(url: URL) -> WebSocketTaskBox { + let task = self.webSocketTask(with: url) + // Avoid "Message too long" receive errors for large snapshots / history payloads. + task.maximumMessageSize = 16 * 1024 * 1024 // 16 MB + return WebSocketTaskBox(task: task) + } +} + +public struct WebSocketSessionBox: @unchecked Sendable { + public let session: any WebSocketSessioning + + public init(session: any WebSocketSessioning) { + self.session = session + } +} + +public struct GatewayConnectOptions: Sendable { + public var role: String + public var scopes: [String] + public var caps: [String] + public var commands: [String] + public var permissions: [String: Bool] + public var clientId: String + public var clientMode: String + public var clientDisplayName: String? + + public init( + role: String, + scopes: [String], + caps: [String], + commands: [String], + permissions: [String: Bool], + clientId: String, + clientMode: String, + clientDisplayName: String?) + { + self.role = role + self.scopes = scopes + self.caps = caps + self.commands = commands + self.permissions = permissions + self.clientId = clientId + self.clientMode = clientMode + self.clientDisplayName = clientDisplayName + } +} + +public enum GatewayAuthSource: String, Sendable { + case deviceToken = "device-token" + case sharedToken = "shared-token" + case password = "password" + case none = "none" +} + +// Avoid ambiguity with the app's own AnyCodable type. +private typealias ProtoAnyCodable = OpenClawProtocol.AnyCodable + +private enum ConnectChallengeError: Error { + case timeout +} + +public actor GatewayChannelActor { + private let logger = Logger(subsystem: "ai.openclaw", category: "gateway") + private var task: WebSocketTaskBox? + private var pending: [String: CheckedContinuation] = [:] + private var connected = false + private var isConnecting = false + private var connectWaiters: [CheckedContinuation] = [] + private var url: URL + private var token: String? + private var password: String? + private let session: WebSocketSessioning + private var backoffMs: Double = 500 + private var shouldReconnect = true + private var lastSeq: Int? + private var lastTick: Date? + private var tickIntervalMs: Double = 30000 + private var lastAuthSource: GatewayAuthSource = .none + private let decoder = JSONDecoder() + private let encoder = JSONEncoder() + private let connectTimeoutSeconds: Double = 6 + private let connectChallengeTimeoutSeconds: Double = 0.75 + private var watchdogTask: Task? + private var tickTask: Task? + private let defaultRequestTimeoutMs: Double = 15000 + private let pushHandler: (@Sendable (GatewayPush) async -> Void)? + private let connectOptions: GatewayConnectOptions? + private let disconnectHandler: (@Sendable (String) async -> Void)? + + public init( + url: URL, + token: String?, + password: String? = nil, + session: WebSocketSessionBox? = nil, + pushHandler: (@Sendable (GatewayPush) async -> Void)? = nil, + connectOptions: GatewayConnectOptions? = nil, + disconnectHandler: (@Sendable (String) async -> Void)? = nil) + { + self.url = url + self.token = token + self.password = password + self.session = session?.session ?? URLSession(configuration: .default) + self.pushHandler = pushHandler + self.connectOptions = connectOptions + self.disconnectHandler = disconnectHandler + Task { [weak self] in + await self?.startWatchdog() + } + } + + public func authSource() -> GatewayAuthSource { self.lastAuthSource } + + public func shutdown() async { + self.shouldReconnect = false + self.connected = false + + self.watchdogTask?.cancel() + self.watchdogTask = nil + + self.tickTask?.cancel() + self.tickTask = nil + + self.task?.cancel(with: .goingAway, reason: nil) + self.task = nil + + await self.failPending(NSError( + domain: "Gateway", + code: 0, + userInfo: [NSLocalizedDescriptionKey: "gateway channel shutdown"])) + + let waiters = self.connectWaiters + self.connectWaiters.removeAll() + for waiter in waiters { + waiter.resume(throwing: NSError( + domain: "Gateway", + code: 0, + userInfo: [NSLocalizedDescriptionKey: "gateway channel shutdown"])) + } + } + + private func startWatchdog() { + self.watchdogTask?.cancel() + self.watchdogTask = Task { [weak self] in + guard let self else { return } + await self.watchdogLoop() + } + } + + private func watchdogLoop() async { + // Keep nudging reconnect in case exponential backoff stalls. + while self.shouldReconnect { + try? await Task.sleep(nanoseconds: 30 * 1_000_000_000) // 30s cadence + guard self.shouldReconnect else { return } + if self.connected { continue } + do { + try await self.connect() + } catch { + let wrapped = self.wrap(error, context: "gateway watchdog reconnect") + self.logger.error("gateway watchdog reconnect failed \(wrapped.localizedDescription, privacy: .public)") + } + } + } + + public func connect() async throws { + if self.connected, self.task?.state == .running { return } + if self.isConnecting { + try await withCheckedThrowingContinuation { cont in + self.connectWaiters.append(cont) + } + return + } + self.isConnecting = true + defer { self.isConnecting = false } + + self.task?.cancel(with: .goingAway, reason: nil) + self.task = self.session.makeWebSocketTask(url: self.url) + self.task?.resume() + do { + try await AsyncTimeout.withTimeout( + seconds: self.connectTimeoutSeconds, + onTimeout: { + NSError( + domain: "Gateway", + code: 1, + userInfo: [NSLocalizedDescriptionKey: "connect timed out"]) + }, + operation: { try await self.sendConnect() }) + } catch { + let wrapped = self.wrap(error, context: "connect to gateway @ \(self.url.absoluteString)") + self.connected = false + self.task?.cancel(with: .goingAway, reason: nil) + await self.disconnectHandler?("connect failed: \(wrapped.localizedDescription)") + let waiters = self.connectWaiters + self.connectWaiters.removeAll() + for waiter in waiters { + waiter.resume(throwing: wrapped) + } + self.logger.error("gateway ws connect failed \(wrapped.localizedDescription, privacy: .public)") + throw wrapped + } + self.listen() + self.connected = true + self.backoffMs = 500 + self.lastSeq = nil + + let waiters = self.connectWaiters + self.connectWaiters.removeAll() + for waiter in waiters { + waiter.resume(returning: ()) + } + } + + private func sendConnect() async throws { + let platform = InstanceIdentity.platformString + let primaryLocale = Locale.preferredLanguages.first ?? Locale.current.identifier + let options = self.connectOptions ?? GatewayConnectOptions( + role: "operator", + scopes: ["operator.admin", "operator.approvals", "operator.pairing"], + caps: [], + commands: [], + permissions: [:], + clientId: "openclaw-macos", + clientMode: "ui", + clientDisplayName: InstanceIdentity.displayName) + let clientDisplayName = options.clientDisplayName ?? InstanceIdentity.displayName + let clientId = options.clientId + let clientMode = options.clientMode + let role = options.role + let scopes = options.scopes + + let reqId = UUID().uuidString + var client: [String: ProtoAnyCodable] = [ + "id": ProtoAnyCodable(clientId), + "displayName": ProtoAnyCodable(clientDisplayName), + "version": ProtoAnyCodable( + Bundle.main.infoDictionary?["CFBundleShortVersionString"] as? String ?? "dev"), + "platform": ProtoAnyCodable(platform), + "mode": ProtoAnyCodable(clientMode), + "instanceId": ProtoAnyCodable(InstanceIdentity.instanceId), + ] + client["deviceFamily"] = ProtoAnyCodable(InstanceIdentity.deviceFamily) + if let model = InstanceIdentity.modelIdentifier { + client["modelIdentifier"] = ProtoAnyCodable(model) + } + var params: [String: ProtoAnyCodable] = [ + "minProtocol": ProtoAnyCodable(GATEWAY_PROTOCOL_VERSION), + "maxProtocol": ProtoAnyCodable(GATEWAY_PROTOCOL_VERSION), + "client": ProtoAnyCodable(client), + "caps": ProtoAnyCodable(options.caps), + "locale": ProtoAnyCodable(primaryLocale), + "userAgent": ProtoAnyCodable(ProcessInfo.processInfo.operatingSystemVersionString), + "role": ProtoAnyCodable(role), + "scopes": ProtoAnyCodable(scopes), + ] + if !options.commands.isEmpty { + params["commands"] = ProtoAnyCodable(options.commands) + } + if !options.permissions.isEmpty { + params["permissions"] = ProtoAnyCodable(options.permissions) + } + let identity = DeviceIdentityStore.loadOrCreate() + let storedToken = DeviceAuthStore.loadToken(deviceId: identity.deviceId, role: role)?.token + let authToken = storedToken ?? self.token + let authSource: GatewayAuthSource + if storedToken != nil { + authSource = .deviceToken + } else if authToken != nil { + authSource = .sharedToken + } else if self.password != nil { + authSource = .password + } else { + authSource = .none + } + self.lastAuthSource = authSource + self.logger.info("gateway connect auth=\(authSource.rawValue, privacy: .public)") + let canFallbackToShared = storedToken != nil && self.token != nil + if let authToken { + params["auth"] = ProtoAnyCodable(["token": ProtoAnyCodable(authToken)]) + } else if let password = self.password { + params["auth"] = ProtoAnyCodable(["password": ProtoAnyCodable(password)]) + } + let signedAtMs = Int(Date().timeIntervalSince1970 * 1000) + let connectNonce = try await self.waitForConnectChallenge() + let scopesValue = scopes.joined(separator: ",") + var payloadParts = [ + connectNonce == nil ? "v1" : "v2", + identity.deviceId, + clientId, + clientMode, + role, + scopesValue, + String(signedAtMs), + authToken ?? "", + ] + if let connectNonce { + payloadParts.append(connectNonce) + } + let payload = payloadParts.joined(separator: "|") + if let signature = DeviceIdentityStore.signPayload(payload, identity: identity), + let publicKey = DeviceIdentityStore.publicKeyBase64Url(identity) { + var device: [String: ProtoAnyCodable] = [ + "id": ProtoAnyCodable(identity.deviceId), + "publicKey": ProtoAnyCodable(publicKey), + "signature": ProtoAnyCodable(signature), + "signedAt": ProtoAnyCodable(signedAtMs), + ] + if let connectNonce { + device["nonce"] = ProtoAnyCodable(connectNonce) + } + params["device"] = ProtoAnyCodable(device) + } + + let frame = RequestFrame( + type: "req", + id: reqId, + method: "connect", + params: ProtoAnyCodable(params)) + let data = try self.encoder.encode(frame) + try await self.task?.send(.data(data)) + do { + let response = try await self.waitForConnectResponse(reqId: reqId) + try await self.handleConnectResponse(response, identity: identity, role: role) + } catch { + if canFallbackToShared { + DeviceAuthStore.clearToken(deviceId: identity.deviceId, role: role) + } + throw error + } + } + + private func handleConnectResponse( + _ res: ResponseFrame, + identity: DeviceIdentity, + role: String + ) async throws { + if res.ok == false { + let msg = (res.error?["message"]?.value as? String) ?? "gateway connect failed" + throw NSError(domain: "Gateway", code: 1008, userInfo: [NSLocalizedDescriptionKey: msg]) + } + guard let payload = res.payload else { + throw NSError( + domain: "Gateway", + code: 1, + userInfo: [NSLocalizedDescriptionKey: "connect failed (missing payload)"]) + } + let payloadData = try self.encoder.encode(payload) + let ok = try decoder.decode(HelloOk.self, from: payloadData) + if let tick = ok.policy["tickIntervalMs"]?.value as? Double { + self.tickIntervalMs = tick + } else if let tick = ok.policy["tickIntervalMs"]?.value as? Int { + self.tickIntervalMs = Double(tick) + } + if let auth = ok.auth, + let deviceToken = auth["deviceToken"]?.value as? String { + let authRole = auth["role"]?.value as? String ?? role + let scopes = (auth["scopes"]?.value as? [ProtoAnyCodable])? + .compactMap { $0.value as? String } ?? [] + _ = DeviceAuthStore.storeToken( + deviceId: identity.deviceId, + role: authRole, + token: deviceToken, + scopes: scopes) + } + self.lastTick = Date() + self.tickTask?.cancel() + self.tickTask = Task { [weak self] in + guard let self else { return } + await self.watchTicks() + } + await self.pushHandler?(.snapshot(ok)) + } + + private func listen() { + self.task?.receive { [weak self] result in + guard let self else { return } + switch result { + case let .failure(err): + Task { await self.handleReceiveFailure(err) } + case let .success(msg): + Task { + await self.handle(msg) + await self.listen() + } + } + } + } + + private func handleReceiveFailure(_ err: Error) async { + let wrapped = self.wrap(err, context: "gateway receive") + self.logger.error("gateway ws receive failed \(wrapped.localizedDescription, privacy: .public)") + self.connected = false + await self.disconnectHandler?("receive failed: \(wrapped.localizedDescription)") + await self.failPending(wrapped) + await self.scheduleReconnect() + } + + private func handle(_ msg: URLSessionWebSocketTask.Message) async { + let data: Data? = switch msg { + case let .data(d): d + case let .string(s): s.data(using: .utf8) + @unknown default: nil + } + guard let data else { return } + guard let frame = try? self.decoder.decode(GatewayFrame.self, from: data) else { + self.logger.error("gateway decode failed") + return + } + switch frame { + case let .res(res): + let id = res.id + if let waiter = pending.removeValue(forKey: id) { + waiter.resume(returning: .res(res)) + } + case let .event(evt): + if evt.event == "connect.challenge" { return } + if let seq = evt.seq { + if let last = lastSeq, seq > last + 1 { + await self.pushHandler?(.seqGap(expected: last + 1, received: seq)) + } + self.lastSeq = seq + } + if evt.event == "tick" { self.lastTick = Date() } + await self.pushHandler?(.event(evt)) + default: + break + } + } + + private func waitForConnectChallenge() async throws -> String? { + guard let task = self.task else { return nil } + do { + return try await AsyncTimeout.withTimeout( + seconds: self.connectChallengeTimeoutSeconds, + onTimeout: { ConnectChallengeError.timeout }, + operation: { [weak self] in + guard let self else { return nil } + while true { + let msg = try await task.receive() + guard let data = self.decodeMessageData(msg) else { continue } + guard let frame = try? self.decoder.decode(GatewayFrame.self, from: data) else { continue } + if case let .event(evt) = frame, evt.event == "connect.challenge" { + if let payload = evt.payload?.value as? [String: ProtoAnyCodable], + let nonce = payload["nonce"]?.value as? String { + return nonce + } + } + } + }) + } catch { + if error is ConnectChallengeError { return nil } + throw error + } + } + + private func waitForConnectResponse(reqId: String) async throws -> ResponseFrame { + guard let task = self.task else { + throw NSError( + domain: "Gateway", + code: 1, + userInfo: [NSLocalizedDescriptionKey: "connect failed (no response)"]) + } + while true { + let msg = try await task.receive() + guard let data = self.decodeMessageData(msg) else { continue } + guard let frame = try? self.decoder.decode(GatewayFrame.self, from: data) else { + throw NSError( + domain: "Gateway", + code: 1, + userInfo: [NSLocalizedDescriptionKey: "connect failed (invalid response)"]) + } + if case let .res(res) = frame, res.id == reqId { + return res + } + } + } + + private nonisolated func decodeMessageData(_ msg: URLSessionWebSocketTask.Message) -> Data? { + let data: Data? = switch msg { + case let .data(data): data + case let .string(text): text.data(using: .utf8) + @unknown default: nil + } + return data + } + + private func watchTicks() async { + let tolerance = self.tickIntervalMs * 2 + while self.connected { + try? await Task.sleep(nanoseconds: UInt64(tolerance * 1_000_000)) + guard self.connected else { return } + if let last = self.lastTick { + let delta = Date().timeIntervalSince(last) * 1000 + if delta > tolerance { + self.logger.error("gateway tick missed; reconnecting") + self.connected = false + await self.failPending( + NSError( + domain: "Gateway", + code: 4, + userInfo: [NSLocalizedDescriptionKey: "gateway tick missed; reconnecting"])) + await self.scheduleReconnect() + return + } + } + } + } + + private func scheduleReconnect() async { + guard self.shouldReconnect else { return } + let delay = self.backoffMs / 1000 + self.backoffMs = min(self.backoffMs * 2, 30000) + try? await Task.sleep(nanoseconds: UInt64(delay * 1_000_000_000)) + guard self.shouldReconnect else { return } + do { + try await self.connect() + } catch { + let wrapped = self.wrap(error, context: "gateway reconnect") + self.logger.error("gateway reconnect failed \(wrapped.localizedDescription, privacy: .public)") + await self.scheduleReconnect() + } + } + + public func request( + method: String, + params: [String: AnyCodable]?, + timeoutMs: Double? = nil) async throws -> Data + { + try await self.connectOrThrow(context: "gateway connect") + let effectiveTimeout = timeoutMs ?? self.defaultRequestTimeoutMs + let payload = try self.encodeRequest(method: method, params: params, kind: "request") + let response = try await withCheckedThrowingContinuation { (cont: CheckedContinuation) in + self.pending[payload.id] = cont + Task { [weak self] in + guard let self else { return } + try? await Task.sleep(nanoseconds: UInt64(effectiveTimeout * 1_000_000)) + await self.timeoutRequest(id: payload.id, timeoutMs: effectiveTimeout) + } + Task { + do { + try await self.task?.send(.data(payload.data)) + } catch { + let wrapped = self.wrap(error, context: "gateway send \(method)") + let waiter = self.pending.removeValue(forKey: payload.id) + // Treat send failures as a broken socket: mark disconnected and trigger reconnect. + self.connected = false + self.task?.cancel(with: .goingAway, reason: nil) + Task { [weak self] in + guard let self else { return } + await self.scheduleReconnect() + } + if let waiter { waiter.resume(throwing: wrapped) } + } + } + } + guard case let .res(res) = response else { + throw NSError(domain: "Gateway", code: 2, userInfo: [NSLocalizedDescriptionKey: "unexpected frame"]) + } + if res.ok == false { + let code = res.error?["code"]?.value as? String + let msg = res.error?["message"]?.value as? String + let details: [String: AnyCodable] = (res.error ?? [:]).reduce(into: [:]) { acc, pair in + acc[pair.key] = AnyCodable(pair.value.value) + } + throw GatewayResponseError(method: method, code: code, message: msg, details: details) + } + if let payload = res.payload { + // Encode back to JSON with Swift's encoder to preserve types and avoid ObjC bridging exceptions. + return try self.encoder.encode(payload) + } + return Data() // Should not happen, but tolerate empty payloads. + } + + public func send(method: String, params: [String: AnyCodable]?) async throws { + try await self.connectOrThrow(context: "gateway connect") + let payload = try self.encodeRequest(method: method, params: params, kind: "send") + guard let task = self.task else { + throw NSError( + domain: "Gateway", + code: 5, + userInfo: [NSLocalizedDescriptionKey: "gateway socket unavailable"]) + } + do { + try await task.send(.data(payload.data)) + } catch { + let wrapped = self.wrap(error, context: "gateway send \(method)") + self.connected = false + self.task?.cancel(with: .goingAway, reason: nil) + Task { [weak self] in + guard let self else { return } + await self.scheduleReconnect() + } + throw wrapped + } + } + + // Wrap low-level URLSession/WebSocket errors with context so UI can surface them. + private func wrap(_ error: Error, context: String) -> Error { + if let urlError = error as? URLError { + let desc = urlError.localizedDescription.isEmpty ? "cancelled" : urlError.localizedDescription + return NSError( + domain: URLError.errorDomain, + code: urlError.errorCode, + userInfo: [NSLocalizedDescriptionKey: "\(context): \(desc)"]) + } + let ns = error as NSError + let desc = ns.localizedDescription.isEmpty ? "unknown" : ns.localizedDescription + return NSError(domain: ns.domain, code: ns.code, userInfo: [NSLocalizedDescriptionKey: "\(context): \(desc)"]) + } + + private func connectOrThrow(context: String) async throws { + do { + try await self.connect() + } catch { + throw self.wrap(error, context: context) + } + } + + private func encodeRequest( + method: String, + params: [String: AnyCodable]?, + kind: String) throws -> (id: String, data: Data) + { + let id = UUID().uuidString + // Encode request using the generated models to avoid JSONSerialization/ObjC bridging pitfalls. + let paramsObject: ProtoAnyCodable? = params.map { entries in + let dict = entries.reduce(into: [String: ProtoAnyCodable]()) { dict, entry in + dict[entry.key] = ProtoAnyCodable(entry.value.value) + } + return ProtoAnyCodable(dict) + } + let frame = RequestFrame( + type: "req", + id: id, + method: method, + params: paramsObject) + do { + let data = try self.encoder.encode(frame) + return (id: id, data: data) + } catch { + self.logger.error( + "gateway \(kind) encode failed \(method, privacy: .public) error=\(error.localizedDescription, privacy: .public)") + throw error + } + } + + private func failPending(_ error: Error) async { + let waiters = self.pending + self.pending.removeAll() + for (_, waiter) in waiters { + waiter.resume(throwing: error) + } + } + + private func timeoutRequest(id: String, timeoutMs: Double) async { + guard let waiter = self.pending.removeValue(forKey: id) else { return } + let err = NSError( + domain: "Gateway", + code: 5, + userInfo: [NSLocalizedDescriptionKey: "gateway request timed out after \(Int(timeoutMs))ms"]) + waiter.resume(throwing: err) + } +} + +// Intentionally no `GatewayChannel` wrapper: the app should use the single shared `GatewayConnection`. diff --git a/apps/shared/OpenClawKit/Sources/OpenClawKit/GatewayEndpointID.swift b/apps/shared/OpenClawKit/Sources/OpenClawKit/GatewayEndpointID.swift new file mode 100644 index 0000000000000000000000000000000000000000..eb2e94f51f419a25239cefccdda8618b4574cf6c --- /dev/null +++ b/apps/shared/OpenClawKit/Sources/OpenClawKit/GatewayEndpointID.swift @@ -0,0 +1,25 @@ +import Foundation +import Network + +public enum GatewayEndpointID { + public static func stableID(_ endpoint: NWEndpoint) -> String { + switch endpoint { + case let .service(name, type, domain, _): + // Keep stable across encoded/decoded differences (e.g. \032 for spaces). + let normalizedName = Self.normalizeServiceNameForID(name) + return "\(type)|\(domain)|\(normalizedName)" + default: + return String(describing: endpoint) + } + } + + public static func prettyDescription(_ endpoint: NWEndpoint) -> String { + BonjourEscapes.decode(String(describing: endpoint)) + } + + private static func normalizeServiceNameForID(_ rawName: String) -> String { + let decoded = BonjourEscapes.decode(rawName) + let normalized = decoded.split(whereSeparator: \.isWhitespace).joined(separator: " ") + return normalized.trimmingCharacters(in: .whitespacesAndNewlines) + } +} diff --git a/apps/shared/OpenClawKit/Sources/OpenClawKit/GatewayErrors.swift b/apps/shared/OpenClawKit/Sources/OpenClawKit/GatewayErrors.swift new file mode 100644 index 0000000000000000000000000000000000000000..6ca81dec44542fe109b23ee54a3e0a878c82c665 --- /dev/null +++ b/apps/shared/OpenClawKit/Sources/OpenClawKit/GatewayErrors.swift @@ -0,0 +1,38 @@ +import OpenClawProtocol +import Foundation + +/// Structured error surfaced when the gateway responds with `{ ok: false }`. +public struct GatewayResponseError: LocalizedError, @unchecked Sendable { + public let method: String + public let code: String + public let message: String + public let details: [String: AnyCodable] + + public init(method: String, code: String?, message: String?, details: [String: AnyCodable]?) { + self.method = method + self.code = (code?.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty == false) + ? code!.trimmingCharacters(in: .whitespacesAndNewlines) + : "GATEWAY_ERROR" + self.message = (message?.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty == false) + ? message!.trimmingCharacters(in: .whitespacesAndNewlines) + : "gateway error" + self.details = details ?? [:] + } + + public var errorDescription: String? { + if self.code == "GATEWAY_ERROR" { return "\(self.method): \(self.message)" } + return "\(self.method): [\(self.code)] \(self.message)" + } +} + +public struct GatewayDecodingError: LocalizedError, Sendable { + public let method: String + public let message: String + + public init(method: String, message: String) { + self.method = method + self.message = message + } + + public var errorDescription: String? { "\(self.method): \(self.message)" } +} diff --git a/apps/shared/OpenClawKit/Sources/OpenClawKit/GatewayNodeSession.swift b/apps/shared/OpenClawKit/Sources/OpenClawKit/GatewayNodeSession.swift new file mode 100644 index 0000000000000000000000000000000000000000..39190f7b88187b0f297b0e7447e48d66bf127fed --- /dev/null +++ b/apps/shared/OpenClawKit/Sources/OpenClawKit/GatewayNodeSession.swift @@ -0,0 +1,262 @@ +import OpenClawProtocol +import Foundation +import OSLog + +private struct NodeInvokeRequestPayload: Codable, Sendable { + var id: String + var nodeId: String + var command: String + var paramsJSON: String? + var timeoutMs: Int? + var idempotencyKey: String? +} + +public actor GatewayNodeSession { + private let logger = Logger(subsystem: "ai.openclaw", category: "node.gateway") + private let decoder = JSONDecoder() + private let encoder = JSONEncoder() + private var channel: GatewayChannelActor? + private var activeURL: URL? + private var activeToken: String? + private var activePassword: String? + private var connectOptions: GatewayConnectOptions? + private var onConnected: (@Sendable () async -> Void)? + private var onDisconnected: (@Sendable (String) async -> Void)? + private var onInvoke: (@Sendable (BridgeInvokeRequest) async -> BridgeInvokeResponse)? + + static func invokeWithTimeout( + request: BridgeInvokeRequest, + timeoutMs: Int?, + onInvoke: @escaping @Sendable (BridgeInvokeRequest) async -> BridgeInvokeResponse + ) async -> BridgeInvokeResponse { + let timeout = max(0, timeoutMs ?? 0) + guard timeout > 0 else { + return await onInvoke(request) + } + + return await withTaskGroup(of: BridgeInvokeResponse.self) { group in + group.addTask { await onInvoke(request) } + group.addTask { + try? await Task.sleep(nanoseconds: UInt64(timeout) * 1_000_000) + return BridgeInvokeResponse( + id: request.id, + ok: false, + error: OpenClawNodeError( + code: .unavailable, + message: "node invoke timed out") + ) + } + + let first = await group.next()! + group.cancelAll() + return first + } + } + private var serverEventSubscribers: [UUID: AsyncStream.Continuation] = [:] + private var canvasHostUrl: String? + + public init() {} + + public func connect( + url: URL, + token: String?, + password: String?, + connectOptions: GatewayConnectOptions, + sessionBox: WebSocketSessionBox?, + onConnected: @escaping @Sendable () async -> Void, + onDisconnected: @escaping @Sendable (String) async -> Void, + onInvoke: @escaping @Sendable (BridgeInvokeRequest) async -> BridgeInvokeResponse + ) async throws { + let shouldReconnect = self.activeURL != url || + self.activeToken != token || + self.activePassword != password || + self.channel == nil + + self.connectOptions = connectOptions + self.onConnected = onConnected + self.onDisconnected = onDisconnected + self.onInvoke = onInvoke + + if shouldReconnect { + if let existing = self.channel { + await existing.shutdown() + } + let channel = GatewayChannelActor( + url: url, + token: token, + password: password, + session: sessionBox, + pushHandler: { [weak self] push in + await self?.handlePush(push) + }, + connectOptions: connectOptions, + disconnectHandler: { [weak self] reason in + await self?.onDisconnected?(reason) + }) + self.channel = channel + self.activeURL = url + self.activeToken = token + self.activePassword = password + } + + guard let channel = self.channel else { + throw NSError(domain: "Gateway", code: 0, userInfo: [ + NSLocalizedDescriptionKey: "gateway channel unavailable", + ]) + } + + do { + try await channel.connect() + await onConnected() + } catch { + await onDisconnected(error.localizedDescription) + throw error + } + } + + public func disconnect() async { + await self.channel?.shutdown() + self.channel = nil + self.activeURL = nil + self.activeToken = nil + self.activePassword = nil + } + + public func currentCanvasHostUrl() -> String? { + self.canvasHostUrl + } + + public func currentRemoteAddress() -> String? { + guard let url = self.activeURL else { return nil } + guard let host = url.host else { return url.absoluteString } + let port = url.port ?? (url.scheme == "wss" ? 443 : 80) + if host.contains(":") { + return "[\(host)]:\(port)" + } + return "\(host):\(port)" + } + + public func sendEvent(event: String, payloadJSON: String?) async { + guard let channel = self.channel else { return } + let params: [String: AnyCodable] = [ + "event": AnyCodable(event), + "payloadJSON": AnyCodable(payloadJSON ?? NSNull()), + ] + do { + try await channel.send(method: "node.event", params: params) + } catch { + self.logger.error("node event failed: \(error.localizedDescription, privacy: .public)") + } + } + + public func request(method: String, paramsJSON: String?, timeoutSeconds: Int = 15) async throws -> Data { + guard let channel = self.channel else { + throw NSError(domain: "Gateway", code: 11, userInfo: [ + NSLocalizedDescriptionKey: "not connected", + ]) + } + + let params = try self.decodeParamsJSON(paramsJSON) + return try await channel.request( + method: method, + params: params, + timeoutMs: Double(timeoutSeconds * 1000)) + } + + public func subscribeServerEvents(bufferingNewest: Int = 200) -> AsyncStream { + let id = UUID() + let session = self + return AsyncStream(bufferingPolicy: .bufferingNewest(bufferingNewest)) { continuation in + self.serverEventSubscribers[id] = continuation + continuation.onTermination = { @Sendable _ in + Task { await session.removeServerEventSubscriber(id) } + } + } + } + + private func handlePush(_ push: GatewayPush) async { + switch push { + case let .snapshot(ok): + let raw = ok.canvashosturl?.trimmingCharacters(in: .whitespacesAndNewlines) + self.canvasHostUrl = (raw?.isEmpty == false) ? raw : nil + await self.onConnected?() + case let .event(evt): + await self.handleEvent(evt) + default: + break + } + } + + private func handleEvent(_ evt: EventFrame) async { + self.broadcastServerEvent(evt) + guard evt.event == "node.invoke.request" else { return } + guard let payload = evt.payload else { return } + do { + let data = try self.encoder.encode(payload) + let request = try self.decoder.decode(NodeInvokeRequestPayload.self, from: data) + guard let onInvoke else { return } + let req = BridgeInvokeRequest(id: request.id, command: request.command, paramsJSON: request.paramsJSON) + let response = await Self.invokeWithTimeout( + request: req, + timeoutMs: request.timeoutMs, + onInvoke: onInvoke + ) + await self.sendInvokeResult(request: request, response: response) + } catch { + self.logger.error("node invoke decode failed: \(error.localizedDescription, privacy: .public)") + } + } + + private func sendInvokeResult(request: NodeInvokeRequestPayload, response: BridgeInvokeResponse) async { + guard let channel = self.channel else { return } + var params: [String: AnyCodable] = [ + "id": AnyCodable(request.id), + "nodeId": AnyCodable(request.nodeId), + "ok": AnyCodable(response.ok), + ] + if let payloadJSON = response.payloadJSON { + params["payloadJSON"] = AnyCodable(payloadJSON) + } + if let error = response.error { + params["error"] = AnyCodable([ + "code": error.code.rawValue, + "message": error.message, + ]) + } + do { + try await channel.send(method: "node.invoke.result", params: params) + } catch { + self.logger.error("node invoke result failed: \(error.localizedDescription, privacy: .public)") + } + } + + private func decodeParamsJSON( + _ paramsJSON: String?) throws -> [String: AnyCodable]? + { + guard let paramsJSON, !paramsJSON.isEmpty else { return nil } + guard let data = paramsJSON.data(using: .utf8) else { + throw NSError(domain: "Gateway", code: 12, userInfo: [ + NSLocalizedDescriptionKey: "paramsJSON not UTF-8", + ]) + } + let raw = try JSONSerialization.jsonObject(with: data) + guard let dict = raw as? [String: Any] else { + return nil + } + return dict.reduce(into: [:]) { acc, entry in + acc[entry.key] = AnyCodable(entry.value) + } + } + + private func broadcastServerEvent(_ evt: EventFrame) { + for (id, continuation) in self.serverEventSubscribers { + if case .terminated = continuation.yield(evt) { + self.serverEventSubscribers.removeValue(forKey: id) + } + } + } + + private func removeServerEventSubscriber(_ id: UUID) { + self.serverEventSubscribers.removeValue(forKey: id) + } +} diff --git a/apps/shared/OpenClawKit/Sources/OpenClawKit/GatewayPayloadDecoding.swift b/apps/shared/OpenClawKit/Sources/OpenClawKit/GatewayPayloadDecoding.swift new file mode 100644 index 0000000000000000000000000000000000000000..8672ab09f681f0ac809160708938d8800cb50360 --- /dev/null +++ b/apps/shared/OpenClawKit/Sources/OpenClawKit/GatewayPayloadDecoding.swift @@ -0,0 +1,36 @@ +import OpenClawProtocol +import Foundation + +public enum GatewayPayloadDecoding { + public static func decode( + _ payload: OpenClawProtocol.AnyCodable, + as _: T.Type = T.self) throws -> T + { + let data = try JSONEncoder().encode(payload) + return try JSONDecoder().decode(T.self, from: data) + } + + public static func decode( + _ payload: AnyCodable, + as _: T.Type = T.self) throws -> T + { + let data = try JSONEncoder().encode(payload) + return try JSONDecoder().decode(T.self, from: data) + } + + public static func decodeIfPresent( + _ payload: OpenClawProtocol.AnyCodable?, + as _: T.Type = T.self) throws -> T? + { + guard let payload else { return nil } + return try self.decode(payload, as: T.self) + } + + public static func decodeIfPresent( + _ payload: AnyCodable?, + as _: T.Type = T.self) throws -> T? + { + guard let payload else { return nil } + return try self.decode(payload, as: T.self) + } +} diff --git a/apps/shared/OpenClawKit/Sources/OpenClawKit/GatewayPush.swift b/apps/shared/OpenClawKit/Sources/OpenClawKit/GatewayPush.swift new file mode 100644 index 0000000000000000000000000000000000000000..65e118ff14ee422bb00454037e801635ff9a41d9 --- /dev/null +++ b/apps/shared/OpenClawKit/Sources/OpenClawKit/GatewayPush.swift @@ -0,0 +1,13 @@ +import OpenClawProtocol + +/// Server-push messages from the gateway websocket. +/// +/// This is the in-process replacement for the legacy `NotificationCenter` fan-out. +public enum GatewayPush: Sendable { + /// A full snapshot that arrives on connect (or reconnect). + case snapshot(HelloOk) + /// A server push event frame. + case event(EventFrame) + /// A detected sequence gap (`expected...received`) for event frames. + case seqGap(expected: Int, received: Int) +} diff --git a/apps/shared/OpenClawKit/Sources/OpenClawKit/GatewayTLSPinning.swift b/apps/shared/OpenClawKit/Sources/OpenClawKit/GatewayTLSPinning.swift new file mode 100644 index 0000000000000000000000000000000000000000..a0cbcd375f61f8abbca7d67fce1b79b05c28c8f5 --- /dev/null +++ b/apps/shared/OpenClawKit/Sources/OpenClawKit/GatewayTLSPinning.swift @@ -0,0 +1,119 @@ +import CryptoKit +import Foundation +import Security + +public struct GatewayTLSParams: Sendable { + public let required: Bool + public let expectedFingerprint: String? + public let allowTOFU: Bool + public let storeKey: String? + + public init(required: Bool, expectedFingerprint: String?, allowTOFU: Bool, storeKey: String?) { + self.required = required + self.expectedFingerprint = expectedFingerprint + self.allowTOFU = allowTOFU + self.storeKey = storeKey + } +} + +public enum GatewayTLSStore { + private static let suiteName = "ai.openclaw.shared" + private static let keyPrefix = "gateway.tls." + + private static var defaults: UserDefaults { + UserDefaults(suiteName: suiteName) ?? .standard + } + + public static func loadFingerprint(stableID: String) -> String? { + let key = self.keyPrefix + stableID + let raw = self.defaults.string(forKey: key)?.trimmingCharacters(in: .whitespacesAndNewlines) + if raw?.isEmpty == false { return raw } + return nil + } + + public static func saveFingerprint(_ value: String, stableID: String) { + let key = self.keyPrefix + stableID + self.defaults.set(value, forKey: key) + } +} + +public final class GatewayTLSPinningSession: NSObject, WebSocketSessioning, URLSessionDelegate, @unchecked Sendable { + private let params: GatewayTLSParams + private lazy var session: URLSession = { + let config = URLSessionConfiguration.default + config.waitsForConnectivity = true + return URLSession(configuration: config, delegate: self, delegateQueue: nil) + }() + + public init(params: GatewayTLSParams) { + self.params = params + super.init() + } + + public func makeWebSocketTask(url: URL) -> WebSocketTaskBox { + let task = self.session.webSocketTask(with: url) + task.maximumMessageSize = 16 * 1024 * 1024 + return WebSocketTaskBox(task: task) + } + + public func urlSession( + _ session: URLSession, + didReceive challenge: URLAuthenticationChallenge, + completionHandler: @escaping (URLSession.AuthChallengeDisposition, URLCredential?) -> Void + ) { + guard challenge.protectionSpace.authenticationMethod == NSURLAuthenticationMethodServerTrust, + let trust = challenge.protectionSpace.serverTrust + else { + completionHandler(.performDefaultHandling, nil) + return + } + + let expected = params.expectedFingerprint.map(normalizeFingerprint) + if let fingerprint = certificateFingerprint(trust) { + if let expected { + if fingerprint == expected { + completionHandler(.useCredential, URLCredential(trust: trust)) + } else { + completionHandler(.cancelAuthenticationChallenge, nil) + } + return + } + if params.allowTOFU { + if let storeKey = params.storeKey { + GatewayTLSStore.saveFingerprint(fingerprint, stableID: storeKey) + } + completionHandler(.useCredential, URLCredential(trust: trust)) + return + } + } + + let ok = SecTrustEvaluateWithError(trust, nil) + if ok || !params.required { + completionHandler(.useCredential, URLCredential(trust: trust)) + } else { + completionHandler(.cancelAuthenticationChallenge, nil) + } + } +} + +private func certificateFingerprint(_ trust: SecTrust) -> String? { + guard let chain = SecTrustCopyCertificateChain(trust) as? [SecCertificate], + let cert = chain.first + else { + return nil + } + return sha256Hex(SecCertificateCopyData(cert) as Data) +} + +private func sha256Hex(_ data: Data) -> String { + let digest = SHA256.hash(data: data) + return digest.map { String(format: "%02x", $0) }.joined() +} + +private func normalizeFingerprint(_ raw: String) -> String { + let stripped = raw.replacingOccurrences( + of: #"(?i)^sha-?256\s*:?\s*"#, + with: "", + options: .regularExpression) + return stripped.lowercased().filter(\.isHexDigit) +} diff --git a/apps/shared/OpenClawKit/Sources/OpenClawKit/InstanceIdentity.swift b/apps/shared/OpenClawKit/Sources/OpenClawKit/InstanceIdentity.swift new file mode 100644 index 0000000000000000000000000000000000000000..d18fa4e9fbf044f6522ee42c3069dca3deaada1a --- /dev/null +++ b/apps/shared/OpenClawKit/Sources/OpenClawKit/InstanceIdentity.swift @@ -0,0 +1,108 @@ +import Foundation + +#if canImport(UIKit) +import UIKit +#endif + +public enum InstanceIdentity { + private static let suiteName = "ai.openclaw.shared" + private static let instanceIdKey = "instanceId" + + private static var defaults: UserDefaults { + UserDefaults(suiteName: suiteName) ?? .standard + } + +#if canImport(UIKit) + private static func readMainActor(_ body: @MainActor () -> T) -> T { + if Thread.isMainThread { + return MainActor.assumeIsolated { body() } + } + return DispatchQueue.main.sync { + MainActor.assumeIsolated { body() } + } + } +#endif + + public static let instanceId: String = { + let defaults = Self.defaults + if let existing = defaults.string(forKey: instanceIdKey)? + .trimmingCharacters(in: .whitespacesAndNewlines), + !existing.isEmpty + { + return existing + } + + let id = UUID().uuidString.lowercased() + defaults.set(id, forKey: instanceIdKey) + return id + }() + + public static let displayName: String = { +#if canImport(UIKit) + let name = Self.readMainActor { + UIDevice.current.name.trimmingCharacters(in: .whitespacesAndNewlines) + } + return name.isEmpty ? "openclaw" : name +#else + if let name = Host.current().localizedName?.trimmingCharacters(in: .whitespacesAndNewlines), + !name.isEmpty + { + return name + } + return "openclaw" +#endif + }() + + public static let modelIdentifier: String? = { +#if canImport(UIKit) + var systemInfo = utsname() + uname(&systemInfo) + let machine = withUnsafeBytes(of: &systemInfo.machine) { ptr in + String(bytes: ptr.prefix { $0 != 0 }, encoding: .utf8) + } + let trimmed = machine?.trimmingCharacters(in: .whitespacesAndNewlines) ?? "" + return trimmed.isEmpty ? nil : trimmed +#else + var size = 0 + guard sysctlbyname("hw.model", nil, &size, nil, 0) == 0, size > 1 else { return nil } + + var buffer = [CChar](repeating: 0, count: size) + guard sysctlbyname("hw.model", &buffer, &size, nil, 0) == 0 else { return nil } + + let bytes = buffer.prefix { $0 != 0 }.map { UInt8(bitPattern: $0) } + guard let raw = String(bytes: bytes, encoding: .utf8) else { return nil } + let trimmed = raw.trimmingCharacters(in: .whitespacesAndNewlines) + return trimmed.isEmpty ? nil : trimmed +#endif + }() + + public static let deviceFamily: String = { +#if canImport(UIKit) + return Self.readMainActor { + switch UIDevice.current.userInterfaceIdiom { + case .pad: return "iPad" + case .phone: return "iPhone" + default: return "iOS" + } + } +#else + return "Mac" +#endif + }() + + public static let platformString: String = { + let v = ProcessInfo.processInfo.operatingSystemVersion +#if canImport(UIKit) + let name = Self.readMainActor { + switch UIDevice.current.userInterfaceIdiom { + case .pad: return "iPadOS" + case .phone: return "iOS" + default: return "iOS" + } + } + return "\(name) \(v.majorVersion).\(v.minorVersion).\(v.patchVersion)" +#else + return "macOS \(v.majorVersion).\(v.minorVersion).\(v.patchVersion)" +#endif + }() +} diff --git a/apps/shared/OpenClawKit/Sources/OpenClawKit/JPEGTranscoder.swift b/apps/shared/OpenClawKit/Sources/OpenClawKit/JPEGTranscoder.swift new file mode 100644 index 0000000000000000000000000000000000000000..f4b1cb95125bdb90724e79f27561e54e1e6e18f4 --- /dev/null +++ b/apps/shared/OpenClawKit/Sources/OpenClawKit/JPEGTranscoder.swift @@ -0,0 +1,135 @@ +import CoreGraphics +import Foundation +import ImageIO +import UniformTypeIdentifiers + +public enum JPEGTranscodeError: LocalizedError, Sendable { + case decodeFailed + case propertiesMissing + case encodeFailed + case sizeLimitExceeded(maxBytes: Int, actualBytes: Int) + + public var errorDescription: String? { + switch self { + case .decodeFailed: + "Failed to decode image data" + case .propertiesMissing: + "Failed to read image properties" + case .encodeFailed: + "Failed to encode JPEG" + case let .sizeLimitExceeded(maxBytes, actualBytes): + "JPEG exceeds size limit (\(actualBytes) bytes > \(maxBytes) bytes)" + } + } +} + +public struct JPEGTranscoder: Sendable { + public static func clampQuality(_ quality: Double) -> Double { + min(1.0, max(0.05, quality)) + } + + /// Re-encodes image data to JPEG, optionally downscaling so that the *oriented* pixel width is <= `maxWidthPx`. + /// + /// - Important: This normalizes EXIF orientation (the output pixels are rotated if needed; orientation tag is not + /// relied on). + public static func transcodeToJPEG( + imageData: Data, + maxWidthPx: Int?, + quality: Double, + maxBytes: Int? = nil) throws -> (data: Data, widthPx: Int, heightPx: Int) + { + guard let src = CGImageSourceCreateWithData(imageData as CFData, nil) else { + throw JPEGTranscodeError.decodeFailed + } + guard + let props = CGImageSourceCopyPropertiesAtIndex(src, 0, nil) as? [CFString: Any], + let rawWidth = props[kCGImagePropertyPixelWidth] as? NSNumber, + let rawHeight = props[kCGImagePropertyPixelHeight] as? NSNumber + else { + throw JPEGTranscodeError.propertiesMissing + } + + let pixelWidth = rawWidth.intValue + let pixelHeight = rawHeight.intValue + let orientation = (props[kCGImagePropertyOrientation] as? NSNumber)?.intValue ?? 1 + + guard pixelWidth > 0, pixelHeight > 0 else { + throw JPEGTranscodeError.propertiesMissing + } + + let rotates90 = orientation == 5 || orientation == 6 || orientation == 7 || orientation == 8 + let orientedWidth = rotates90 ? pixelHeight : pixelWidth + let orientedHeight = rotates90 ? pixelWidth : pixelHeight + + let maxDim = max(orientedWidth, orientedHeight) + var targetMaxPixelSize: Int = { + guard let maxWidthPx, maxWidthPx > 0 else { return maxDim } + guard orientedWidth > maxWidthPx else { return maxDim } // never upscale + + let scale = Double(maxWidthPx) / Double(orientedWidth) + return max(1, Int((Double(maxDim) * scale).rounded(.toNearestOrAwayFromZero))) + }() + + func encode(maxPixelSize: Int, quality: Double) throws -> (data: Data, widthPx: Int, heightPx: Int) { + let thumbOpts: [CFString: Any] = [ + kCGImageSourceCreateThumbnailFromImageAlways: true, + kCGImageSourceCreateThumbnailWithTransform: true, + kCGImageSourceThumbnailMaxPixelSize: maxPixelSize, + kCGImageSourceShouldCacheImmediately: true, + ] + + guard let img = CGImageSourceCreateThumbnailAtIndex(src, 0, thumbOpts as CFDictionary) else { + throw JPEGTranscodeError.decodeFailed + } + + let out = NSMutableData() + guard let dest = CGImageDestinationCreateWithData(out, UTType.jpeg.identifier as CFString, 1, nil) else { + throw JPEGTranscodeError.encodeFailed + } + let q = self.clampQuality(quality) + let encodeProps = [kCGImageDestinationLossyCompressionQuality: q] as CFDictionary + CGImageDestinationAddImage(dest, img, encodeProps) + guard CGImageDestinationFinalize(dest) else { + throw JPEGTranscodeError.encodeFailed + } + + return (out as Data, img.width, img.height) + } + + guard let maxBytes, maxBytes > 0 else { + return try encode(maxPixelSize: targetMaxPixelSize, quality: quality) + } + + let minQuality = max(0.2, self.clampQuality(quality) * 0.35) + let minPixelSize = 256 + var best = try encode(maxPixelSize: targetMaxPixelSize, quality: quality) + if best.data.count <= maxBytes { + return best + } + + for _ in 0..<6 { + var q = self.clampQuality(quality) + for _ in 0..<6 { + let candidate = try encode(maxPixelSize: targetMaxPixelSize, quality: q) + best = candidate + if candidate.data.count <= maxBytes { + return candidate + } + if q <= minQuality { break } + q = max(minQuality, q * 0.75) + } + + let nextPixelSize = max(Int(Double(targetMaxPixelSize) * 0.85), minPixelSize) + if nextPixelSize == targetMaxPixelSize { + break + } + targetMaxPixelSize = nextPixelSize + } + + if best.data.count > maxBytes { + throw JPEGTranscodeError.sizeLimitExceeded(maxBytes: maxBytes, actualBytes: best.data.count) + } + + return best + } +} diff --git a/apps/shared/OpenClawKit/Sources/OpenClawKit/LocationCommands.swift b/apps/shared/OpenClawKit/Sources/OpenClawKit/LocationCommands.swift new file mode 100644 index 0000000000000000000000000000000000000000..c02bc84202d656b6f617901555e796ff98ba1777 --- /dev/null +++ b/apps/shared/OpenClawKit/Sources/OpenClawKit/LocationCommands.swift @@ -0,0 +1,57 @@ +import Foundation + +public enum OpenClawLocationCommand: String, Codable, Sendable { + case get = "location.get" +} + +public enum OpenClawLocationAccuracy: String, Codable, Sendable { + case coarse + case balanced + case precise +} + +public struct OpenClawLocationGetParams: Codable, Sendable, Equatable { + public var timeoutMs: Int? + public var maxAgeMs: Int? + public var desiredAccuracy: OpenClawLocationAccuracy? + + public init(timeoutMs: Int? = nil, maxAgeMs: Int? = nil, desiredAccuracy: OpenClawLocationAccuracy? = nil) { + self.timeoutMs = timeoutMs + self.maxAgeMs = maxAgeMs + self.desiredAccuracy = desiredAccuracy + } +} + +public struct OpenClawLocationPayload: Codable, Sendable, Equatable { + public var lat: Double + public var lon: Double + public var accuracyMeters: Double + public var altitudeMeters: Double? + public var speedMps: Double? + public var headingDeg: Double? + public var timestamp: String + public var isPrecise: Bool + public var source: String? + + public init( + lat: Double, + lon: Double, + accuracyMeters: Double, + altitudeMeters: Double? = nil, + speedMps: Double? = nil, + headingDeg: Double? = nil, + timestamp: String, + isPrecise: Bool, + source: String? = nil) + { + self.lat = lat + self.lon = lon + self.accuracyMeters = accuracyMeters + self.altitudeMeters = altitudeMeters + self.speedMps = speedMps + self.headingDeg = headingDeg + self.timestamp = timestamp + self.isPrecise = isPrecise + self.source = source + } +} diff --git a/apps/shared/OpenClawKit/Sources/OpenClawKit/LocationSettings.swift b/apps/shared/OpenClawKit/Sources/OpenClawKit/LocationSettings.swift new file mode 100644 index 0000000000000000000000000000000000000000..961e2980c5191d8f048173e728261e79a8dd4026 --- /dev/null +++ b/apps/shared/OpenClawKit/Sources/OpenClawKit/LocationSettings.swift @@ -0,0 +1,7 @@ +import Foundation + +public enum OpenClawLocationMode: String, Codable, Sendable, CaseIterable { + case off + case whileUsing + case always +} diff --git a/apps/shared/OpenClawKit/Sources/OpenClawKit/NodeError.swift b/apps/shared/OpenClawKit/Sources/OpenClawKit/NodeError.swift new file mode 100644 index 0000000000000000000000000000000000000000..4fe3fd042aea1ff1de4bce03a82e580be912b6f6 --- /dev/null +++ b/apps/shared/OpenClawKit/Sources/OpenClawKit/NodeError.swift @@ -0,0 +1,28 @@ +import Foundation + +public enum OpenClawNodeErrorCode: String, Codable, Sendable { + case notPaired = "NOT_PAIRED" + case unauthorized = "UNAUTHORIZED" + case backgroundUnavailable = "NODE_BACKGROUND_UNAVAILABLE" + case invalidRequest = "INVALID_REQUEST" + case unavailable = "UNAVAILABLE" +} + +public struct OpenClawNodeError: Error, Codable, Sendable, Equatable { + public var code: OpenClawNodeErrorCode + public var message: String + public var retryable: Bool? + public var retryAfterMs: Int? + + public init( + code: OpenClawNodeErrorCode, + message: String, + retryable: Bool? = nil, + retryAfterMs: Int? = nil) + { + self.code = code + self.message = message + self.retryable = retryable + self.retryAfterMs = retryAfterMs + } +} diff --git a/apps/shared/OpenClawKit/Sources/OpenClawKit/OpenClawKitResources.swift b/apps/shared/OpenClawKit/Sources/OpenClawKit/OpenClawKitResources.swift new file mode 100644 index 0000000000000000000000000000000000000000..b19792ad7b813e4bed2a3475aea31ad4eb8664f1 --- /dev/null +++ b/apps/shared/OpenClawKit/Sources/OpenClawKit/OpenClawKitResources.swift @@ -0,0 +1,75 @@ +import Foundation + +public enum OpenClawKitResources { + /// Resource bundle for OpenClawKit. + /// + /// Locates the SwiftPM-generated resource bundle, checking multiple locations: + /// 1. Inside Bundle.main (packaged apps) + /// 2. Bundle.module (SwiftPM development/tests) + /// 3. Falls back to Bundle.main if not found (resource lookups will return nil) + /// + /// This avoids a fatal crash when Bundle.module can't locate its resources + /// in packaged .app bundles where the resource bundle path differs from + /// SwiftPM's expectations. + public static let bundle: Bundle = locateBundle() + + private static let bundleName = "OpenClawKit_OpenClawKit" + + private static func locateBundle() -> Bundle { + // 1. Check inside Bundle.main (packaged apps copy resources here) + if let mainResourceURL = Bundle.main.resourceURL { + let bundleURL = mainResourceURL.appendingPathComponent("\(bundleName).bundle") + if let bundle = Bundle(url: bundleURL) { + return bundle + } + } + + // 2. Check Bundle.main directly for embedded resources + if Bundle.main.url(forResource: "tool-display", withExtension: "json") != nil { + return Bundle.main + } + + // 3. Try Bundle.module (works in SwiftPM development/tests) + // Wrap in a function to defer the fatalError until actually called + if let moduleBundle = loadModuleBundleSafely() { + return moduleBundle + } + + // 4. Fallback: return Bundle.main (resource lookups will return nil gracefully) + return Bundle.main + } + + private static func loadModuleBundleSafely() -> Bundle? { + // Bundle.module is generated by SwiftPM and will fatalError if not found. + // We check likely locations manually to avoid the crash. + let candidates: [URL?] = [ + Bundle.main.resourceURL, + Bundle.main.bundleURL, + Bundle(for: BundleLocator.self).resourceURL, + Bundle(for: BundleLocator.self).bundleURL, + ] + + for candidate in candidates { + guard let baseURL = candidate else { continue } + + // Direct path + let directURL = baseURL.appendingPathComponent("\(bundleName).bundle") + if let bundle = Bundle(url: directURL) { + return bundle + } + + // Inside Resources/ + let resourcesURL = baseURL + .appendingPathComponent("Resources") + .appendingPathComponent("\(bundleName).bundle") + if let bundle = Bundle(url: resourcesURL) { + return bundle + } + } + + return nil + } +} + +// Helper class for bundle lookup via Bundle(for:) +private final class BundleLocator {} diff --git a/apps/shared/OpenClawKit/Sources/OpenClawKit/Resources/CanvasScaffold/scaffold.html b/apps/shared/OpenClawKit/Sources/OpenClawKit/Resources/CanvasScaffold/scaffold.html new file mode 100644 index 0000000000000000000000000000000000000000..ceb7a975da43d2fe0a680900301b7321d14d3009 --- /dev/null +++ b/apps/shared/OpenClawKit/Sources/OpenClawKit/Resources/CanvasScaffold/scaffold.html @@ -0,0 +1,225 @@ + + + + + + Canvas + + + + + +
+
+
Ready
+
Waiting for agent
+
+
+ + + diff --git a/apps/shared/OpenClawKit/Sources/OpenClawKit/Resources/tool-display.json b/apps/shared/OpenClawKit/Sources/OpenClawKit/Resources/tool-display.json new file mode 100644 index 0000000000000000000000000000000000000000..9c0e57fc6ae7898e7d07bad906a2408d3b2737af --- /dev/null +++ b/apps/shared/OpenClawKit/Sources/OpenClawKit/Resources/tool-display.json @@ -0,0 +1,197 @@ +{ + "version": 1, + "fallback": { + "emoji": "🧩", + "detailKeys": [ + "command", + "path", + "url", + "targetUrl", + "targetId", + "ref", + "element", + "node", + "nodeId", + "id", + "requestId", + "to", + "channelId", + "guildId", + "userId", + "name", + "query", + "pattern", + "messageId" + ] + }, + "tools": { + "bash": { + "emoji": "🛠️", + "title": "Bash", + "detailKeys": ["command"] + }, + "process": { + "emoji": "🧰", + "title": "Process", + "detailKeys": ["sessionId"] + }, + "read": { + "emoji": "📖", + "title": "Read", + "detailKeys": ["path"] + }, + "write": { + "emoji": "✍️", + "title": "Write", + "detailKeys": ["path"] + }, + "edit": { + "emoji": "📝", + "title": "Edit", + "detailKeys": ["path"] + }, + "attach": { + "emoji": "📎", + "title": "Attach", + "detailKeys": ["path", "url", "fileName"] + }, + "browser": { + "emoji": "🌐", + "title": "Browser", + "actions": { + "status": { "label": "status" }, + "start": { "label": "start" }, + "stop": { "label": "stop" }, + "tabs": { "label": "tabs" }, + "open": { "label": "open", "detailKeys": ["targetUrl"] }, + "focus": { "label": "focus", "detailKeys": ["targetId"] }, + "close": { "label": "close", "detailKeys": ["targetId"] }, + "snapshot": { + "label": "snapshot", + "detailKeys": ["targetUrl", "targetId", "ref", "element", "format"] + }, + "screenshot": { + "label": "screenshot", + "detailKeys": ["targetUrl", "targetId", "ref", "element"] + }, + "navigate": { + "label": "navigate", + "detailKeys": ["targetUrl", "targetId"] + }, + "console": { "label": "console", "detailKeys": ["level", "targetId"] }, + "pdf": { "label": "pdf", "detailKeys": ["targetId"] }, + "upload": { + "label": "upload", + "detailKeys": ["paths", "ref", "inputRef", "element", "targetId"] + }, + "dialog": { + "label": "dialog", + "detailKeys": ["accept", "promptText", "targetId"] + }, + "act": { + "label": "act", + "detailKeys": ["request.kind", "request.ref", "request.selector", "request.text", "request.value"] + } + } + }, + "canvas": { + "emoji": "🖼️", + "title": "Canvas", + "actions": { + "present": { "label": "present", "detailKeys": ["target", "node", "nodeId"] }, + "hide": { "label": "hide", "detailKeys": ["node", "nodeId"] }, + "navigate": { "label": "navigate", "detailKeys": ["url", "node", "nodeId"] }, + "eval": { "label": "eval", "detailKeys": ["javaScript", "node", "nodeId"] }, + "snapshot": { "label": "snapshot", "detailKeys": ["format", "node", "nodeId"] }, + "a2ui_push": { "label": "A2UI push", "detailKeys": ["jsonlPath", "node", "nodeId"] }, + "a2ui_reset": { "label": "A2UI reset", "detailKeys": ["node", "nodeId"] } + } + }, + "nodes": { + "emoji": "📱", + "title": "Nodes", + "actions": { + "status": { "label": "status" }, + "describe": { "label": "describe", "detailKeys": ["node", "nodeId"] }, + "pending": { "label": "pending" }, + "approve": { "label": "approve", "detailKeys": ["requestId"] }, + "reject": { "label": "reject", "detailKeys": ["requestId"] }, + "notify": { "label": "notify", "detailKeys": ["node", "nodeId", "title", "body"] }, + "camera_snap": { "label": "camera snap", "detailKeys": ["node", "nodeId", "facing", "deviceId"] }, + "camera_list": { "label": "camera list", "detailKeys": ["node", "nodeId"] }, + "camera_clip": { "label": "camera clip", "detailKeys": ["node", "nodeId", "facing", "duration", "durationMs"] }, + "screen_record": { + "label": "screen record", + "detailKeys": ["node", "nodeId", "duration", "durationMs", "fps", "screenIndex"] + } + } + }, + "cron": { + "emoji": "⏰", + "title": "Cron", + "actions": { + "status": { "label": "status" }, + "list": { "label": "list" }, + "add": { + "label": "add", + "detailKeys": ["job.name", "job.id", "job.schedule", "job.cron"] + }, + "update": { "label": "update", "detailKeys": ["id"] }, + "remove": { "label": "remove", "detailKeys": ["id"] }, + "run": { "label": "run", "detailKeys": ["id"] }, + "runs": { "label": "runs", "detailKeys": ["id"] }, + "wake": { "label": "wake", "detailKeys": ["text", "mode"] } + } + }, + "gateway": { + "emoji": "🔌", + "title": "Gateway", + "actions": { + "restart": { "label": "restart", "detailKeys": ["reason", "delayMs"] } + } + }, + "whatsapp_login": { + "emoji": "🟢", + "title": "WhatsApp Login", + "actions": { + "start": { "label": "start" }, + "wait": { "label": "wait" } + } + }, + "discord": { + "emoji": "💬", + "title": "Discord", + "actions": { + "react": { "label": "react", "detailKeys": ["channelId", "messageId", "emoji"] }, + "reactions": { "label": "reactions", "detailKeys": ["channelId", "messageId"] }, + "sticker": { "label": "sticker", "detailKeys": ["to", "stickerIds"] }, + "poll": { "label": "poll", "detailKeys": ["question", "to"] }, + "permissions": { "label": "permissions", "detailKeys": ["channelId"] }, + "readMessages": { "label": "read messages", "detailKeys": ["channelId", "limit"] }, + "sendMessage": { "label": "send", "detailKeys": ["to", "content"] }, + "editMessage": { "label": "edit", "detailKeys": ["channelId", "messageId"] }, + "deleteMessage": { "label": "delete", "detailKeys": ["channelId", "messageId"] }, + "threadCreate": { "label": "thread create", "detailKeys": ["channelId", "name"] }, + "threadList": { "label": "thread list", "detailKeys": ["guildId", "channelId"] }, + "threadReply": { "label": "thread reply", "detailKeys": ["channelId", "content"] }, + "pinMessage": { "label": "pin", "detailKeys": ["channelId", "messageId"] }, + "unpinMessage": { "label": "unpin", "detailKeys": ["channelId", "messageId"] }, + "listPins": { "label": "list pins", "detailKeys": ["channelId"] }, + "searchMessages": { "label": "search", "detailKeys": ["guildId", "content"] }, + "memberInfo": { "label": "member", "detailKeys": ["guildId", "userId"] }, + "roleInfo": { "label": "roles", "detailKeys": ["guildId"] }, + "emojiList": { "label": "emoji list", "detailKeys": ["guildId"] }, + "roleAdd": { "label": "role add", "detailKeys": ["guildId", "userId", "roleId"] }, + "roleRemove": { "label": "role remove", "detailKeys": ["guildId", "userId", "roleId"] }, + "channelInfo": { "label": "channel", "detailKeys": ["channelId"] }, + "channelList": { "label": "channels", "detailKeys": ["guildId"] }, + "voiceStatus": { "label": "voice", "detailKeys": ["guildId", "userId"] }, + "eventList": { "label": "events", "detailKeys": ["guildId"] }, + "eventCreate": { "label": "event create", "detailKeys": ["guildId", "name"] }, + "timeout": { "label": "timeout", "detailKeys": ["guildId", "userId"] }, + "kick": { "label": "kick", "detailKeys": ["guildId", "userId"] }, + "ban": { "label": "ban", "detailKeys": ["guildId", "userId"] } + } + } + } +} diff --git a/apps/shared/OpenClawKit/Sources/OpenClawKit/ScreenCommands.swift b/apps/shared/OpenClawKit/Sources/OpenClawKit/ScreenCommands.swift new file mode 100644 index 0000000000000000000000000000000000000000..dfb57ce2ab245c833ce36b18d6b0f67f482587ed --- /dev/null +++ b/apps/shared/OpenClawKit/Sources/OpenClawKit/ScreenCommands.swift @@ -0,0 +1,27 @@ +import Foundation + +public enum OpenClawScreenCommand: String, Codable, Sendable { + case record = "screen.record" +} + +public struct OpenClawScreenRecordParams: Codable, Sendable, Equatable { + public var screenIndex: Int? + public var durationMs: Int? + public var fps: Double? + public var format: String? + public var includeAudio: Bool? + + public init( + screenIndex: Int? = nil, + durationMs: Int? = nil, + fps: Double? = nil, + format: String? = nil, + includeAudio: Bool? = nil) + { + self.screenIndex = screenIndex + self.durationMs = durationMs + self.fps = fps + self.format = format + self.includeAudio = includeAudio + } +} diff --git a/apps/shared/OpenClawKit/Sources/OpenClawKit/StoragePaths.swift b/apps/shared/OpenClawKit/Sources/OpenClawKit/StoragePaths.swift new file mode 100644 index 0000000000000000000000000000000000000000..d75422957112d2d448b23d96aa5861cfd650ce2d --- /dev/null +++ b/apps/shared/OpenClawKit/Sources/OpenClawKit/StoragePaths.swift @@ -0,0 +1,37 @@ +import Foundation + +public enum OpenClawNodeStorage { + public static func appSupportDir() throws -> URL { + let base = FileManager().urls(for: .applicationSupportDirectory, in: .userDomainMask).first + guard let base else { + throw NSError(domain: "OpenClawNodeStorage", code: 1, userInfo: [ + NSLocalizedDescriptionKey: "Application Support directory unavailable", + ]) + } + return base.appendingPathComponent("OpenClaw", isDirectory: true) + } + + public static func canvasRoot(sessionKey: String) throws -> URL { + let root = try appSupportDir().appendingPathComponent("canvas", isDirectory: true) + let safe = sessionKey.trimmingCharacters(in: .whitespacesAndNewlines) + let session = safe.isEmpty ? "main" : safe + return root.appendingPathComponent(session, isDirectory: true) + } + + public static func cachesDir() throws -> URL { + let base = FileManager().urls(for: .cachesDirectory, in: .userDomainMask).first + guard let base else { + throw NSError(domain: "OpenClawNodeStorage", code: 2, userInfo: [ + NSLocalizedDescriptionKey: "Caches directory unavailable", + ]) + } + return base.appendingPathComponent("OpenClaw", isDirectory: true) + } + + public static func canvasSnapshotsRoot(sessionKey: String) throws -> URL { + let root = try cachesDir().appendingPathComponent("canvas-snapshots", isDirectory: true) + let safe = sessionKey.trimmingCharacters(in: .whitespacesAndNewlines) + let session = safe.isEmpty ? "main" : safe + return root.appendingPathComponent(session, isDirectory: true) + } +} diff --git a/apps/shared/OpenClawKit/Sources/OpenClawKit/SystemCommands.swift b/apps/shared/OpenClawKit/Sources/OpenClawKit/SystemCommands.swift new file mode 100644 index 0000000000000000000000000000000000000000..a2c8349058b4eeff8507a3bc62e4d7d9d8efda6c --- /dev/null +++ b/apps/shared/OpenClawKit/Sources/OpenClawKit/SystemCommands.swift @@ -0,0 +1,88 @@ +import Foundation + +public enum OpenClawSystemCommand: String, Codable, Sendable { + case run = "system.run" + case which = "system.which" + case notify = "system.notify" + case execApprovalsGet = "system.execApprovals.get" + case execApprovalsSet = "system.execApprovals.set" +} + +public enum OpenClawNotificationPriority: String, Codable, Sendable { + case passive + case active + case timeSensitive +} + +public enum OpenClawNotificationDelivery: String, Codable, Sendable { + case system + case overlay + case auto +} + +public struct OpenClawSystemRunParams: Codable, Sendable, Equatable { + public var command: [String] + public var rawCommand: String? + public var cwd: String? + public var env: [String: String]? + public var timeoutMs: Int? + public var needsScreenRecording: Bool? + public var agentId: String? + public var sessionKey: String? + public var approved: Bool? + public var approvalDecision: String? + + public init( + command: [String], + rawCommand: String? = nil, + cwd: String? = nil, + env: [String: String]? = nil, + timeoutMs: Int? = nil, + needsScreenRecording: Bool? = nil, + agentId: String? = nil, + sessionKey: String? = nil, + approved: Bool? = nil, + approvalDecision: String? = nil) + { + self.command = command + self.rawCommand = rawCommand + self.cwd = cwd + self.env = env + self.timeoutMs = timeoutMs + self.needsScreenRecording = needsScreenRecording + self.agentId = agentId + self.sessionKey = sessionKey + self.approved = approved + self.approvalDecision = approvalDecision + } +} + +public struct OpenClawSystemWhichParams: Codable, Sendable, Equatable { + public var bins: [String] + + public init(bins: [String]) { + self.bins = bins + } +} + +public struct OpenClawSystemNotifyParams: Codable, Sendable, Equatable { + public var title: String + public var body: String + public var sound: String? + public var priority: OpenClawNotificationPriority? + public var delivery: OpenClawNotificationDelivery? + + public init( + title: String, + body: String, + sound: String? = nil, + priority: OpenClawNotificationPriority? = nil, + delivery: OpenClawNotificationDelivery? = nil) + { + self.title = title + self.body = body + self.sound = sound + self.priority = priority + self.delivery = delivery + } +} diff --git a/apps/shared/OpenClawKit/Sources/OpenClawKit/TalkDirective.swift b/apps/shared/OpenClawKit/Sources/OpenClawKit/TalkDirective.swift new file mode 100644 index 0000000000000000000000000000000000000000..6c460dc0267e6cd8ef0bd7167d862dadc82582ca --- /dev/null +++ b/apps/shared/OpenClawKit/Sources/OpenClawKit/TalkDirective.swift @@ -0,0 +1,201 @@ +import Foundation + +public struct TalkDirective: Equatable, Sendable { + public var voiceId: String? + public var modelId: String? + public var speed: Double? + public var rateWPM: Int? + public var stability: Double? + public var similarity: Double? + public var style: Double? + public var speakerBoost: Bool? + public var seed: Int? + public var normalize: String? + public var language: String? + public var outputFormat: String? + public var latencyTier: Int? + public var once: Bool? + + public init( + voiceId: String? = nil, + modelId: String? = nil, + speed: Double? = nil, + rateWPM: Int? = nil, + stability: Double? = nil, + similarity: Double? = nil, + style: Double? = nil, + speakerBoost: Bool? = nil, + seed: Int? = nil, + normalize: String? = nil, + language: String? = nil, + outputFormat: String? = nil, + latencyTier: Int? = nil, + once: Bool? = nil) + { + self.voiceId = voiceId + self.modelId = modelId + self.speed = speed + self.rateWPM = rateWPM + self.stability = stability + self.similarity = similarity + self.style = style + self.speakerBoost = speakerBoost + self.seed = seed + self.normalize = normalize + self.language = language + self.outputFormat = outputFormat + self.latencyTier = latencyTier + self.once = once + } +} + +public struct TalkDirectiveParseResult: Equatable, Sendable { + public let directive: TalkDirective? + public let stripped: String + public let unknownKeys: [String] + + public init(directive: TalkDirective?, stripped: String, unknownKeys: [String]) { + self.directive = directive + self.stripped = stripped + self.unknownKeys = unknownKeys + } +} + +public enum TalkDirectiveParser { + public static func parse(_ text: String) -> TalkDirectiveParseResult { + let normalized = text.replacingOccurrences(of: "\r\n", with: "\n") + var lines = normalized.split(separator: "\n", omittingEmptySubsequences: false) + guard !lines.isEmpty else { return TalkDirectiveParseResult(directive: nil, stripped: text, unknownKeys: []) } + + guard let firstNonEmptyIndex = + lines.firstIndex(where: { !$0.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty }) + else { + return TalkDirectiveParseResult(directive: nil, stripped: text, unknownKeys: []) + } + + var firstNonEmpty = firstNonEmptyIndex + if firstNonEmpty > 0 { + lines.removeSubrange(0.. String? { + for key in keys { + if let value = dict[key] as? String { + let trimmed = value.trimmingCharacters(in: .whitespacesAndNewlines) + if !trimmed.isEmpty { return trimmed } + } + } + return nil + } + + private static func doubleValue(_ dict: [String: Any], keys: [String]) -> Double? { + for key in keys { + if let value = dict[key] as? Double { return value } + if let value = dict[key] as? Int { return Double(value) } + if let value = dict[key] as? String, let parsed = Double(value) { return parsed } + } + return nil + } + + private static func intValue(_ dict: [String: Any], keys: [String]) -> Int? { + for key in keys { + if let value = dict[key] as? Int { return value } + if let value = dict[key] as? Double { return Int(value) } + if let value = dict[key] as? String, let parsed = Int(value) { return parsed } + } + return nil + } + + private static func boolValue(_ dict: [String: Any], keys: [String]) -> Bool? { + for key in keys { + if let value = dict[key] as? Bool { return value } + if let value = dict[key] as? String { + let trimmed = value.trimmingCharacters(in: .whitespacesAndNewlines).lowercased() + if ["true", "yes", "1"].contains(trimmed) { return true } + if ["false", "no", "0"].contains(trimmed) { return false } + } + } + return nil + } +} diff --git a/apps/shared/OpenClawKit/Sources/OpenClawKit/TalkHistoryTimestamp.swift b/apps/shared/OpenClawKit/Sources/OpenClawKit/TalkHistoryTimestamp.swift new file mode 100644 index 0000000000000000000000000000000000000000..75f14ef85b4a7073be85ae6457080adb245eb93d --- /dev/null +++ b/apps/shared/OpenClawKit/Sources/OpenClawKit/TalkHistoryTimestamp.swift @@ -0,0 +1,12 @@ +public enum TalkHistoryTimestamp: Sendable { + /// Gateway history timestamps have historically been emitted as either seconds (Double, epoch seconds) + /// or milliseconds (Double, epoch ms). This helper accepts either. + public static func isAfter(_ timestamp: Double, sinceSeconds: Double) -> Bool { + let sinceMs = sinceSeconds * 1000 + // ~2286-11-20 in epoch seconds. Anything bigger is almost certainly epoch milliseconds. + if timestamp > 10_000_000_000 { + return timestamp >= sinceMs - 500 + } + return timestamp >= sinceSeconds - 0.5 + } +} diff --git a/apps/shared/OpenClawKit/Sources/OpenClawKit/TalkPromptBuilder.swift b/apps/shared/OpenClawKit/Sources/OpenClawKit/TalkPromptBuilder.swift new file mode 100644 index 0000000000000000000000000000000000000000..c63f40e9d3a7dc59c849abb247d1a5d493f1da0f --- /dev/null +++ b/apps/shared/OpenClawKit/Sources/OpenClawKit/TalkPromptBuilder.swift @@ -0,0 +1,17 @@ +public enum TalkPromptBuilder: Sendable { + public static func build(transcript: String, interruptedAtSeconds: Double?) -> String { + var lines: [String] = [ + "Talk Mode active. Reply in a concise, spoken tone.", + "You may optionally prefix the response with JSON (first line) to set ElevenLabs voice (id or alias), e.g. {\"voice\":\"\",\"once\":true}.", + ] + + if let interruptedAtSeconds { + let formatted = String(format: "%.1f", interruptedAtSeconds) + lines.append("Assistant speech interrupted at \(formatted)s.") + } + + lines.append("") + lines.append(transcript) + return lines.joined(separator: "\n") + } +} diff --git a/apps/shared/OpenClawKit/Sources/OpenClawKit/TalkSystemSpeechSynthesizer.swift b/apps/shared/OpenClawKit/Sources/OpenClawKit/TalkSystemSpeechSynthesizer.swift new file mode 100644 index 0000000000000000000000000000000000000000..4cfc536da877db0aa015e000e4104ed145f56c95 --- /dev/null +++ b/apps/shared/OpenClawKit/Sources/OpenClawKit/TalkSystemSpeechSynthesizer.swift @@ -0,0 +1,116 @@ +import AVFoundation +import Foundation + +@MainActor +public final class TalkSystemSpeechSynthesizer: NSObject { + public enum SpeakError: Error { + case canceled + } + + public static let shared = TalkSystemSpeechSynthesizer() + + private let synth = AVSpeechSynthesizer() + private var speakContinuation: CheckedContinuation? + private var currentUtterance: AVSpeechUtterance? + private var currentToken = UUID() + private var watchdog: Task? + + public var isSpeaking: Bool { self.synth.isSpeaking } + + override private init() { + super.init() + self.synth.delegate = self + } + + public func stop() { + self.currentToken = UUID() + self.watchdog?.cancel() + self.watchdog = nil + self.synth.stopSpeaking(at: .immediate) + self.finishCurrent(with: SpeakError.canceled) + } + + public func speak(text: String, language: String? = nil) async throws { + let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines) + guard !trimmed.isEmpty else { return } + + self.stop() + let token = UUID() + self.currentToken = token + + let utterance = AVSpeechUtterance(string: trimmed) + if let language, let voice = AVSpeechSynthesisVoice(language: language) { + utterance.voice = voice + } + self.currentUtterance = utterance + + let estimatedSeconds = max(3.0, min(180.0, Double(trimmed.count) * 0.08)) + self.watchdog?.cancel() + self.watchdog = Task { @MainActor [weak self] in + guard let self else { return } + try? await Task.sleep(nanoseconds: UInt64(estimatedSeconds * 1_000_000_000)) + if Task.isCancelled { return } + guard self.currentToken == token else { return } + if self.synth.isSpeaking { + self.synth.stopSpeaking(at: .immediate) + } + self.finishCurrent( + with: NSError(domain: "TalkSystemSpeechSynthesizer", code: 408, userInfo: [ + NSLocalizedDescriptionKey: "system TTS timed out after \(estimatedSeconds)s", + ])) + } + + try await withTaskCancellationHandler(operation: { + try await withCheckedThrowingContinuation { cont in + self.speakContinuation = cont + self.synth.speak(utterance) + } + }, onCancel: { + Task { @MainActor in + self.stop() + } + }) + + if self.currentToken != token { + throw SpeakError.canceled + } + } + + private func handleFinish(error: Error?) { + guard self.currentUtterance != nil else { return } + self.watchdog?.cancel() + self.watchdog = nil + self.finishCurrent(with: error) + } + + private func finishCurrent(with error: Error?) { + self.currentUtterance = nil + let cont = self.speakContinuation + self.speakContinuation = nil + if let error { + cont?.resume(throwing: error) + } else { + cont?.resume(returning: ()) + } + } +} + +extension TalkSystemSpeechSynthesizer: AVSpeechSynthesizerDelegate { + public nonisolated func speechSynthesizer( + _ synthesizer: AVSpeechSynthesizer, + didFinish utterance: AVSpeechUtterance) + { + Task { @MainActor in + self.handleFinish(error: nil) + } + } + + public nonisolated func speechSynthesizer( + _ synthesizer: AVSpeechSynthesizer, + didCancel utterance: AVSpeechUtterance) + { + Task { @MainActor in + self.handleFinish(error: SpeakError.canceled) + } + } +} diff --git a/apps/shared/OpenClawKit/Sources/OpenClawKit/ToolDisplay.swift b/apps/shared/OpenClawKit/Sources/OpenClawKit/ToolDisplay.swift new file mode 100644 index 0000000000000000000000000000000000000000..d52e24ca8560a496e88b7bd1d0d271a8e380299c --- /dev/null +++ b/apps/shared/OpenClawKit/Sources/OpenClawKit/ToolDisplay.swift @@ -0,0 +1,265 @@ +import Foundation + +public struct ToolDisplaySummary: Sendable, Equatable { + public let name: String + public let emoji: String + public let title: String + public let label: String + public let verb: String? + public let detail: String? + + public var detailLine: String? { + var parts: [String] = [] + if let verb, !verb.isEmpty { parts.append(verb) } + if let detail, !detail.isEmpty { parts.append(detail) } + return parts.isEmpty ? nil : parts.joined(separator: " · ") + } + + public var summaryLine: String { + if let detailLine { + return "\(self.emoji) \(self.label): \(detailLine)" + } + return "\(self.emoji) \(self.label)" + } +} + +public enum ToolDisplayRegistry { + private struct ToolDisplayActionSpec: Decodable { + let label: String? + let detailKeys: [String]? + } + + private struct ToolDisplaySpec: Decodable { + let emoji: String? + let title: String? + let label: String? + let detailKeys: [String]? + let actions: [String: ToolDisplayActionSpec]? + } + + private struct ToolDisplayConfig: Decodable { + let version: Int? + let fallback: ToolDisplaySpec? + let tools: [String: ToolDisplaySpec]? + } + + private static let config: ToolDisplayConfig = loadConfig() + + public static func resolve(name: String?, args: AnyCodable?, meta: String? = nil) -> ToolDisplaySummary { + let trimmedName = name?.trimmingCharacters(in: .whitespacesAndNewlines) ?? "tool" + let key = trimmedName.lowercased() + let spec = self.config.tools?[key] + let fallback = self.config.fallback + + let emoji = spec?.emoji ?? fallback?.emoji ?? "🧩" + let title = spec?.title ?? self.titleFromName(trimmedName) + let label = spec?.label ?? trimmedName + + let actionRaw = self.valueForKeyPath(args, path: "action") as? String + let action = actionRaw?.trimmingCharacters(in: .whitespacesAndNewlines) + let actionSpec = action.flatMap { spec?.actions?[$0] } + let verb = self.normalizeVerb(actionSpec?.label ?? action) + + var detail: String? + if key == "read" { + detail = self.readDetail(args) + } else if key == "write" || key == "edit" || key == "attach" { + detail = self.pathDetail(args) + } + + let detailKeys = actionSpec?.detailKeys ?? spec?.detailKeys ?? fallback?.detailKeys ?? [] + if detail == nil { + detail = self.firstValue(args, keys: detailKeys) + } + + if detail == nil { + detail = meta + } + + if let detailValue = detail { + detail = self.shortenHomeInString(detailValue) + } + + return ToolDisplaySummary( + name: trimmedName, + emoji: emoji, + title: title, + label: label, + verb: verb, + detail: detail) + } + + private static func loadConfig() -> ToolDisplayConfig { + guard let url = OpenClawKitResources.bundle.url(forResource: "tool-display", withExtension: "json") else { + return self.defaultConfig() + } + do { + let data = try Data(contentsOf: url) + return try JSONDecoder().decode(ToolDisplayConfig.self, from: data) + } catch { + return self.defaultConfig() + } + } + + private static func defaultConfig() -> ToolDisplayConfig { + ToolDisplayConfig( + version: 1, + fallback: ToolDisplaySpec( + emoji: "🧩", + title: nil, + label: nil, + detailKeys: [ + "command", + "path", + "url", + "targetUrl", + "targetId", + "ref", + "element", + "node", + "nodeId", + "id", + "requestId", + "to", + "channelId", + "guildId", + "userId", + "name", + "query", + "pattern", + "messageId", + ], + actions: nil), + tools: [ + "bash": ToolDisplaySpec( + emoji: "🛠️", + title: "Bash", + label: nil, + detailKeys: ["command"], + actions: nil), + "read": ToolDisplaySpec( + emoji: "📖", + title: "Read", + label: nil, + detailKeys: ["path"], + actions: nil), + "write": ToolDisplaySpec( + emoji: "✍️", + title: "Write", + label: nil, + detailKeys: ["path"], + actions: nil), + "edit": ToolDisplaySpec( + emoji: "📝", + title: "Edit", + label: nil, + detailKeys: ["path"], + actions: nil), + "attach": ToolDisplaySpec( + emoji: "📎", + title: "Attach", + label: nil, + detailKeys: ["path", "url", "fileName"], + actions: nil), + "process": ToolDisplaySpec( + emoji: "🧰", + title: "Process", + label: nil, + detailKeys: ["sessionId"], + actions: nil), + ]) + } + + private static func titleFromName(_ name: String) -> String { + let cleaned = name.replacingOccurrences(of: "_", with: " ").trimmingCharacters(in: .whitespaces) + guard !cleaned.isEmpty else { return "Tool" } + return cleaned + .split(separator: " ") + .map { part in + let upper = part.uppercased() + if part.count <= 2, part == upper { return String(part) } + return String(upper.prefix(1)) + String(part.lowercased().dropFirst()) + } + .joined(separator: " ") + } + + private static func normalizeVerb(_ value: String?) -> String? { + let trimmed = value?.trimmingCharacters(in: .whitespacesAndNewlines) ?? "" + guard !trimmed.isEmpty else { return nil } + return trimmed.replacingOccurrences(of: "_", with: " ") + } + + private static func readDetail(_ args: AnyCodable?) -> String? { + guard let path = valueForKeyPath(args, path: "path") as? String else { return nil } + let offsetAny = self.valueForKeyPath(args, path: "offset") + let limitAny = self.valueForKeyPath(args, path: "limit") + let offset = (offsetAny as? Double) ?? (offsetAny as? Int).map(Double.init) + let limit = (limitAny as? Double) ?? (limitAny as? Int).map(Double.init) + if let offset, let limit { + let end = offset + limit + return "\(path):\(Int(offset))-\(Int(end))" + } + return path + } + + private static func pathDetail(_ args: AnyCodable?) -> String? { + self.valueForKeyPath(args, path: "path") as? String + } + + private static func firstValue(_ args: AnyCodable?, keys: [String]) -> String? { + for key in keys { + if let value = valueForKeyPath(args, path: key), + let rendered = renderValue(value) + { + return rendered + } + } + return nil + } + + private static func renderValue(_ value: Any) -> String? { + if let str = value as? String { + let trimmed = str.trimmingCharacters(in: .whitespacesAndNewlines) + guard !trimmed.isEmpty else { return nil } + let first = trimmed.split(whereSeparator: \.isNewline).first.map(String.init) ?? trimmed + if first.count > 160 { return String(first.prefix(157)) + "…" } + return first + } + if let num = value as? Int { return String(num) } + if let num = value as? Double { return String(num) } + if let bool = value as? Bool { return bool ? "true" : "false" } + if let array = value as? [Any] { + let items = array.compactMap { self.renderValue($0) } + guard !items.isEmpty else { return nil } + let preview = items.prefix(3).joined(separator: ", ") + return items.count > 3 ? "\(preview)…" : preview + } + if let dict = value as? [String: Any] { + if let label = dict["name"].flatMap({ renderValue($0) }) { return label } + if let label = dict["id"].flatMap({ renderValue($0) }) { return label } + } + return nil + } + + private static func valueForKeyPath(_ args: AnyCodable?, path: String) -> Any? { + guard let args else { return nil } + let parts = path.split(separator: ".").map(String.init) + var current: Any? = args.value + for part in parts { + if let dict = current as? [String: AnyCodable] { + current = dict[part]?.value + } else if let dict = current as? [String: Any] { + current = dict[part] + } else { + return nil + } + } + return current + } + + private static func shortenHomeInString(_ value: String) -> String { + let home = NSHomeDirectory() + guard !home.isEmpty else { return value } + return value.replacingOccurrences(of: home, with: "~") + } +} diff --git a/apps/shared/OpenClawKit/Sources/OpenClawProtocol/AnyCodable.swift b/apps/shared/OpenClawKit/Sources/OpenClawProtocol/AnyCodable.swift new file mode 100644 index 0000000000000000000000000000000000000000..ad0c3387296770c8ad5d2aa95096dc1a6e11f7fb --- /dev/null +++ b/apps/shared/OpenClawKit/Sources/OpenClawProtocol/AnyCodable.swift @@ -0,0 +1,54 @@ +import Foundation + +/// Lightweight `Codable` wrapper that round-trips heterogeneous JSON payloads. +/// Marked `@unchecked Sendable` because it can hold reference types. +public struct AnyCodable: Codable, @unchecked Sendable { + public let value: Any + + public init(_ value: Any) { self.value = value } + + public init(from decoder: Decoder) throws { + let container = try decoder.singleValueContainer() + if let intVal = try? container.decode(Int.self) { self.value = intVal; return } + if let doubleVal = try? container.decode(Double.self) { self.value = doubleVal; return } + if let boolVal = try? container.decode(Bool.self) { self.value = boolVal; return } + if let stringVal = try? container.decode(String.self) { self.value = stringVal; return } + if container.decodeNil() { self.value = NSNull(); return } + if let dict = try? container.decode([String: AnyCodable].self) { self.value = dict; return } + if let array = try? container.decode([AnyCodable].self) { self.value = array; return } + throw DecodingError.dataCorruptedError( + in: container, + debugDescription: "Unsupported type") + } + + public func encode(to encoder: Encoder) throws { + var container = encoder.singleValueContainer() + switch self.value { + case let intVal as Int: try container.encode(intVal) + case let doubleVal as Double: try container.encode(doubleVal) + case let boolVal as Bool: try container.encode(boolVal) + case let stringVal as String: try container.encode(stringVal) + case is NSNull: try container.encodeNil() + case let dict as [String: AnyCodable]: try container.encode(dict) + case let array as [AnyCodable]: try container.encode(array) + case let dict as [String: Any]: + try container.encode(dict.mapValues { AnyCodable($0) }) + case let array as [Any]: + try container.encode(array.map { AnyCodable($0) }) + case let dict as NSDictionary: + var converted: [String: AnyCodable] = [:] + for (k, v) in dict { + guard let key = k as? String else { continue } + converted[key] = AnyCodable(v) + } + try container.encode(converted) + case let array as NSArray: + try container.encode(array.map { AnyCodable($0) }) + default: + let context = EncodingError.Context( + codingPath: encoder.codingPath, + debugDescription: "Unsupported type") + throw EncodingError.invalidValue(self.value, context) + } + } +} diff --git a/apps/shared/OpenClawKit/Sources/OpenClawProtocol/GatewayModels.swift b/apps/shared/OpenClawKit/Sources/OpenClawProtocol/GatewayModels.swift new file mode 100644 index 0000000000000000000000000000000000000000..9d2ca5ed4ca6579ae06e15904325a4ea80a8216a --- /dev/null +++ b/apps/shared/OpenClawKit/Sources/OpenClawProtocol/GatewayModels.swift @@ -0,0 +1,2454 @@ +// Generated by scripts/protocol-gen-swift.ts — do not edit by hand +import Foundation + +public let GATEWAY_PROTOCOL_VERSION = 3 + +public enum ErrorCode: String, Codable, Sendable { + case notLinked = "NOT_LINKED" + case notPaired = "NOT_PAIRED" + case agentTimeout = "AGENT_TIMEOUT" + case invalidRequest = "INVALID_REQUEST" + case unavailable = "UNAVAILABLE" +} + +public struct ConnectParams: Codable, Sendable { + public let minprotocol: Int + public let maxprotocol: Int + public let client: [String: AnyCodable] + public let caps: [String]? + public let commands: [String]? + public let permissions: [String: AnyCodable]? + public let pathenv: String? + public let role: String? + public let scopes: [String]? + public let device: [String: AnyCodable]? + public let auth: [String: AnyCodable]? + public let locale: String? + public let useragent: String? + + public init( + minprotocol: Int, + maxprotocol: Int, + client: [String: AnyCodable], + caps: [String]?, + commands: [String]?, + permissions: [String: AnyCodable]?, + pathenv: String?, + role: String?, + scopes: [String]?, + device: [String: AnyCodable]?, + auth: [String: AnyCodable]?, + locale: String?, + useragent: String? + ) { + self.minprotocol = minprotocol + self.maxprotocol = maxprotocol + self.client = client + self.caps = caps + self.commands = commands + self.permissions = permissions + self.pathenv = pathenv + self.role = role + self.scopes = scopes + self.device = device + self.auth = auth + self.locale = locale + self.useragent = useragent + } + private enum CodingKeys: String, CodingKey { + case minprotocol = "minProtocol" + case maxprotocol = "maxProtocol" + case client + case caps + case commands + case permissions + case pathenv = "pathEnv" + case role + case scopes + case device + case auth + case locale + case useragent = "userAgent" + } +} + +public struct HelloOk: Codable, Sendable { + public let type: String + public let _protocol: Int + public let server: [String: AnyCodable] + public let features: [String: AnyCodable] + public let snapshot: Snapshot + public let canvashosturl: String? + public let auth: [String: AnyCodable]? + public let policy: [String: AnyCodable] + + public init( + type: String, + _protocol: Int, + server: [String: AnyCodable], + features: [String: AnyCodable], + snapshot: Snapshot, + canvashosturl: String?, + auth: [String: AnyCodable]?, + policy: [String: AnyCodable] + ) { + self.type = type + self._protocol = _protocol + self.server = server + self.features = features + self.snapshot = snapshot + self.canvashosturl = canvashosturl + self.auth = auth + self.policy = policy + } + private enum CodingKeys: String, CodingKey { + case type + case _protocol = "protocol" + case server + case features + case snapshot + case canvashosturl = "canvasHostUrl" + case auth + case policy + } +} + +public struct RequestFrame: Codable, Sendable { + public let type: String + public let id: String + public let method: String + public let params: AnyCodable? + + public init( + type: String, + id: String, + method: String, + params: AnyCodable? + ) { + self.type = type + self.id = id + self.method = method + self.params = params + } + private enum CodingKeys: String, CodingKey { + case type + case id + case method + case params + } +} + +public struct ResponseFrame: Codable, Sendable { + public let type: String + public let id: String + public let ok: Bool + public let payload: AnyCodable? + public let error: [String: AnyCodable]? + + public init( + type: String, + id: String, + ok: Bool, + payload: AnyCodable?, + error: [String: AnyCodable]? + ) { + self.type = type + self.id = id + self.ok = ok + self.payload = payload + self.error = error + } + private enum CodingKeys: String, CodingKey { + case type + case id + case ok + case payload + case error + } +} + +public struct EventFrame: Codable, Sendable { + public let type: String + public let event: String + public let payload: AnyCodable? + public let seq: Int? + public let stateversion: [String: AnyCodable]? + + public init( + type: String, + event: String, + payload: AnyCodable?, + seq: Int?, + stateversion: [String: AnyCodable]? + ) { + self.type = type + self.event = event + self.payload = payload + self.seq = seq + self.stateversion = stateversion + } + private enum CodingKeys: String, CodingKey { + case type + case event + case payload + case seq + case stateversion = "stateVersion" + } +} + +public struct PresenceEntry: Codable, Sendable { + public let host: String? + public let ip: String? + public let version: String? + public let platform: String? + public let devicefamily: String? + public let modelidentifier: String? + public let mode: String? + public let lastinputseconds: Int? + public let reason: String? + public let tags: [String]? + public let text: String? + public let ts: Int + public let deviceid: String? + public let roles: [String]? + public let scopes: [String]? + public let instanceid: String? + + public init( + host: String?, + ip: String?, + version: String?, + platform: String?, + devicefamily: String?, + modelidentifier: String?, + mode: String?, + lastinputseconds: Int?, + reason: String?, + tags: [String]?, + text: String?, + ts: Int, + deviceid: String?, + roles: [String]?, + scopes: [String]?, + instanceid: String? + ) { + self.host = host + self.ip = ip + self.version = version + self.platform = platform + self.devicefamily = devicefamily + self.modelidentifier = modelidentifier + self.mode = mode + self.lastinputseconds = lastinputseconds + self.reason = reason + self.tags = tags + self.text = text + self.ts = ts + self.deviceid = deviceid + self.roles = roles + self.scopes = scopes + self.instanceid = instanceid + } + private enum CodingKeys: String, CodingKey { + case host + case ip + case version + case platform + case devicefamily = "deviceFamily" + case modelidentifier = "modelIdentifier" + case mode + case lastinputseconds = "lastInputSeconds" + case reason + case tags + case text + case ts + case deviceid = "deviceId" + case roles + case scopes + case instanceid = "instanceId" + } +} + +public struct StateVersion: Codable, Sendable { + public let presence: Int + public let health: Int + + public init( + presence: Int, + health: Int + ) { + self.presence = presence + self.health = health + } + private enum CodingKeys: String, CodingKey { + case presence + case health + } +} + +public struct Snapshot: Codable, Sendable { + public let presence: [PresenceEntry] + public let health: AnyCodable + public let stateversion: StateVersion + public let uptimems: Int + public let configpath: String? + public let statedir: String? + public let sessiondefaults: [String: AnyCodable]? + + public init( + presence: [PresenceEntry], + health: AnyCodable, + stateversion: StateVersion, + uptimems: Int, + configpath: String?, + statedir: String?, + sessiondefaults: [String: AnyCodable]? + ) { + self.presence = presence + self.health = health + self.stateversion = stateversion + self.uptimems = uptimems + self.configpath = configpath + self.statedir = statedir + self.sessiondefaults = sessiondefaults + } + private enum CodingKeys: String, CodingKey { + case presence + case health + case stateversion = "stateVersion" + case uptimems = "uptimeMs" + case configpath = "configPath" + case statedir = "stateDir" + case sessiondefaults = "sessionDefaults" + } +} + +public struct ErrorShape: Codable, Sendable { + public let code: String + public let message: String + public let details: AnyCodable? + public let retryable: Bool? + public let retryafterms: Int? + + public init( + code: String, + message: String, + details: AnyCodable?, + retryable: Bool?, + retryafterms: Int? + ) { + self.code = code + self.message = message + self.details = details + self.retryable = retryable + self.retryafterms = retryafterms + } + private enum CodingKeys: String, CodingKey { + case code + case message + case details + case retryable + case retryafterms = "retryAfterMs" + } +} + +public struct AgentEvent: Codable, Sendable { + public let runid: String + public let seq: Int + public let stream: String + public let ts: Int + public let data: [String: AnyCodable] + + public init( + runid: String, + seq: Int, + stream: String, + ts: Int, + data: [String: AnyCodable] + ) { + self.runid = runid + self.seq = seq + self.stream = stream + self.ts = ts + self.data = data + } + private enum CodingKeys: String, CodingKey { + case runid = "runId" + case seq + case stream + case ts + case data + } +} + +public struct SendParams: Codable, Sendable { + public let to: String + public let message: String + public let mediaurl: String? + public let mediaurls: [String]? + public let gifplayback: Bool? + public let channel: String? + public let accountid: String? + public let sessionkey: String? + public let idempotencykey: String + + public init( + to: String, + message: String, + mediaurl: String?, + mediaurls: [String]?, + gifplayback: Bool?, + channel: String?, + accountid: String?, + sessionkey: String?, + idempotencykey: String + ) { + self.to = to + self.message = message + self.mediaurl = mediaurl + self.mediaurls = mediaurls + self.gifplayback = gifplayback + self.channel = channel + self.accountid = accountid + self.sessionkey = sessionkey + self.idempotencykey = idempotencykey + } + private enum CodingKeys: String, CodingKey { + case to + case message + case mediaurl = "mediaUrl" + case mediaurls = "mediaUrls" + case gifplayback = "gifPlayback" + case channel + case accountid = "accountId" + case sessionkey = "sessionKey" + case idempotencykey = "idempotencyKey" + } +} + +public struct PollParams: Codable, Sendable { + public let to: String + public let question: String + public let options: [String] + public let maxselections: Int? + public let durationhours: Int? + public let channel: String? + public let accountid: String? + public let idempotencykey: String + + public init( + to: String, + question: String, + options: [String], + maxselections: Int?, + durationhours: Int?, + channel: String?, + accountid: String?, + idempotencykey: String + ) { + self.to = to + self.question = question + self.options = options + self.maxselections = maxselections + self.durationhours = durationhours + self.channel = channel + self.accountid = accountid + self.idempotencykey = idempotencykey + } + private enum CodingKeys: String, CodingKey { + case to + case question + case options + case maxselections = "maxSelections" + case durationhours = "durationHours" + case channel + case accountid = "accountId" + case idempotencykey = "idempotencyKey" + } +} + +public struct AgentParams: Codable, Sendable { + public let message: String + public let agentid: String? + public let to: String? + public let replyto: String? + public let sessionid: String? + public let sessionkey: String? + public let thinking: String? + public let deliver: Bool? + public let attachments: [AnyCodable]? + public let channel: String? + public let replychannel: String? + public let accountid: String? + public let replyaccountid: String? + public let threadid: String? + public let groupid: String? + public let groupchannel: String? + public let groupspace: String? + public let timeout: Int? + public let lane: String? + public let extrasystemprompt: String? + public let idempotencykey: String + public let label: String? + public let spawnedby: String? + + public init( + message: String, + agentid: String?, + to: String?, + replyto: String?, + sessionid: String?, + sessionkey: String?, + thinking: String?, + deliver: Bool?, + attachments: [AnyCodable]?, + channel: String?, + replychannel: String?, + accountid: String?, + replyaccountid: String?, + threadid: String?, + groupid: String?, + groupchannel: String?, + groupspace: String?, + timeout: Int?, + lane: String?, + extrasystemprompt: String?, + idempotencykey: String, + label: String?, + spawnedby: String? + ) { + self.message = message + self.agentid = agentid + self.to = to + self.replyto = replyto + self.sessionid = sessionid + self.sessionkey = sessionkey + self.thinking = thinking + self.deliver = deliver + self.attachments = attachments + self.channel = channel + self.replychannel = replychannel + self.accountid = accountid + self.replyaccountid = replyaccountid + self.threadid = threadid + self.groupid = groupid + self.groupchannel = groupchannel + self.groupspace = groupspace + self.timeout = timeout + self.lane = lane + self.extrasystemprompt = extrasystemprompt + self.idempotencykey = idempotencykey + self.label = label + self.spawnedby = spawnedby + } + private enum CodingKeys: String, CodingKey { + case message + case agentid = "agentId" + case to + case replyto = "replyTo" + case sessionid = "sessionId" + case sessionkey = "sessionKey" + case thinking + case deliver + case attachments + case channel + case replychannel = "replyChannel" + case accountid = "accountId" + case replyaccountid = "replyAccountId" + case threadid = "threadId" + case groupid = "groupId" + case groupchannel = "groupChannel" + case groupspace = "groupSpace" + case timeout + case lane + case extrasystemprompt = "extraSystemPrompt" + case idempotencykey = "idempotencyKey" + case label + case spawnedby = "spawnedBy" + } +} + +public struct AgentIdentityParams: Codable, Sendable { + public let agentid: String? + public let sessionkey: String? + + public init( + agentid: String?, + sessionkey: String? + ) { + self.agentid = agentid + self.sessionkey = sessionkey + } + private enum CodingKeys: String, CodingKey { + case agentid = "agentId" + case sessionkey = "sessionKey" + } +} + +public struct AgentIdentityResult: Codable, Sendable { + public let agentid: String + public let name: String? + public let avatar: String? + + public init( + agentid: String, + name: String?, + avatar: String? + ) { + self.agentid = agentid + self.name = name + self.avatar = avatar + } + private enum CodingKeys: String, CodingKey { + case agentid = "agentId" + case name + case avatar + } +} + +public struct AgentWaitParams: Codable, Sendable { + public let runid: String + public let timeoutms: Int? + + public init( + runid: String, + timeoutms: Int? + ) { + self.runid = runid + self.timeoutms = timeoutms + } + private enum CodingKeys: String, CodingKey { + case runid = "runId" + case timeoutms = "timeoutMs" + } +} + +public struct WakeParams: Codable, Sendable { + public let mode: AnyCodable + public let text: String + + public init( + mode: AnyCodable, + text: String + ) { + self.mode = mode + self.text = text + } + private enum CodingKeys: String, CodingKey { + case mode + case text + } +} + +public struct NodePairRequestParams: Codable, Sendable { + public let nodeid: String + public let displayname: String? + public let platform: String? + public let version: String? + public let coreversion: String? + public let uiversion: String? + public let devicefamily: String? + public let modelidentifier: String? + public let caps: [String]? + public let commands: [String]? + public let remoteip: String? + public let silent: Bool? + + public init( + nodeid: String, + displayname: String?, + platform: String?, + version: String?, + coreversion: String?, + uiversion: String?, + devicefamily: String?, + modelidentifier: String?, + caps: [String]?, + commands: [String]?, + remoteip: String?, + silent: Bool? + ) { + self.nodeid = nodeid + self.displayname = displayname + self.platform = platform + self.version = version + self.coreversion = coreversion + self.uiversion = uiversion + self.devicefamily = devicefamily + self.modelidentifier = modelidentifier + self.caps = caps + self.commands = commands + self.remoteip = remoteip + self.silent = silent + } + private enum CodingKeys: String, CodingKey { + case nodeid = "nodeId" + case displayname = "displayName" + case platform + case version + case coreversion = "coreVersion" + case uiversion = "uiVersion" + case devicefamily = "deviceFamily" + case modelidentifier = "modelIdentifier" + case caps + case commands + case remoteip = "remoteIp" + case silent + } +} + +public struct NodePairListParams: Codable, Sendable { +} + +public struct NodePairApproveParams: Codable, Sendable { + public let requestid: String + + public init( + requestid: String + ) { + self.requestid = requestid + } + private enum CodingKeys: String, CodingKey { + case requestid = "requestId" + } +} + +public struct NodePairRejectParams: Codable, Sendable { + public let requestid: String + + public init( + requestid: String + ) { + self.requestid = requestid + } + private enum CodingKeys: String, CodingKey { + case requestid = "requestId" + } +} + +public struct NodePairVerifyParams: Codable, Sendable { + public let nodeid: String + public let token: String + + public init( + nodeid: String, + token: String + ) { + self.nodeid = nodeid + self.token = token + } + private enum CodingKeys: String, CodingKey { + case nodeid = "nodeId" + case token + } +} + +public struct NodeRenameParams: Codable, Sendable { + public let nodeid: String + public let displayname: String + + public init( + nodeid: String, + displayname: String + ) { + self.nodeid = nodeid + self.displayname = displayname + } + private enum CodingKeys: String, CodingKey { + case nodeid = "nodeId" + case displayname = "displayName" + } +} + +public struct NodeListParams: Codable, Sendable { +} + +public struct NodeDescribeParams: Codable, Sendable { + public let nodeid: String + + public init( + nodeid: String + ) { + self.nodeid = nodeid + } + private enum CodingKeys: String, CodingKey { + case nodeid = "nodeId" + } +} + +public struct NodeInvokeParams: Codable, Sendable { + public let nodeid: String + public let command: String + public let params: AnyCodable? + public let timeoutms: Int? + public let idempotencykey: String + + public init( + nodeid: String, + command: String, + params: AnyCodable?, + timeoutms: Int?, + idempotencykey: String + ) { + self.nodeid = nodeid + self.command = command + self.params = params + self.timeoutms = timeoutms + self.idempotencykey = idempotencykey + } + private enum CodingKeys: String, CodingKey { + case nodeid = "nodeId" + case command + case params + case timeoutms = "timeoutMs" + case idempotencykey = "idempotencyKey" + } +} + +public struct NodeInvokeResultParams: Codable, Sendable { + public let id: String + public let nodeid: String + public let ok: Bool + public let payload: AnyCodable? + public let payloadjson: String? + public let error: [String: AnyCodable]? + + public init( + id: String, + nodeid: String, + ok: Bool, + payload: AnyCodable?, + payloadjson: String?, + error: [String: AnyCodable]? + ) { + self.id = id + self.nodeid = nodeid + self.ok = ok + self.payload = payload + self.payloadjson = payloadjson + self.error = error + } + private enum CodingKeys: String, CodingKey { + case id + case nodeid = "nodeId" + case ok + case payload + case payloadjson = "payloadJSON" + case error + } +} + +public struct NodeEventParams: Codable, Sendable { + public let event: String + public let payload: AnyCodable? + public let payloadjson: String? + + public init( + event: String, + payload: AnyCodable?, + payloadjson: String? + ) { + self.event = event + self.payload = payload + self.payloadjson = payloadjson + } + private enum CodingKeys: String, CodingKey { + case event + case payload + case payloadjson = "payloadJSON" + } +} + +public struct NodeInvokeRequestEvent: Codable, Sendable { + public let id: String + public let nodeid: String + public let command: String + public let paramsjson: String? + public let timeoutms: Int? + public let idempotencykey: String? + + public init( + id: String, + nodeid: String, + command: String, + paramsjson: String?, + timeoutms: Int?, + idempotencykey: String? + ) { + self.id = id + self.nodeid = nodeid + self.command = command + self.paramsjson = paramsjson + self.timeoutms = timeoutms + self.idempotencykey = idempotencykey + } + private enum CodingKeys: String, CodingKey { + case id + case nodeid = "nodeId" + case command + case paramsjson = "paramsJSON" + case timeoutms = "timeoutMs" + case idempotencykey = "idempotencyKey" + } +} + +public struct SessionsListParams: Codable, Sendable { + public let limit: Int? + public let activeminutes: Int? + public let includeglobal: Bool? + public let includeunknown: Bool? + public let includederivedtitles: Bool? + public let includelastmessage: Bool? + public let label: String? + public let spawnedby: String? + public let agentid: String? + public let search: String? + + public init( + limit: Int?, + activeminutes: Int?, + includeglobal: Bool?, + includeunknown: Bool?, + includederivedtitles: Bool?, + includelastmessage: Bool?, + label: String?, + spawnedby: String?, + agentid: String?, + search: String? + ) { + self.limit = limit + self.activeminutes = activeminutes + self.includeglobal = includeglobal + self.includeunknown = includeunknown + self.includederivedtitles = includederivedtitles + self.includelastmessage = includelastmessage + self.label = label + self.spawnedby = spawnedby + self.agentid = agentid + self.search = search + } + private enum CodingKeys: String, CodingKey { + case limit + case activeminutes = "activeMinutes" + case includeglobal = "includeGlobal" + case includeunknown = "includeUnknown" + case includederivedtitles = "includeDerivedTitles" + case includelastmessage = "includeLastMessage" + case label + case spawnedby = "spawnedBy" + case agentid = "agentId" + case search + } +} + +public struct SessionsPreviewParams: Codable, Sendable { + public let keys: [String] + public let limit: Int? + public let maxchars: Int? + + public init( + keys: [String], + limit: Int?, + maxchars: Int? + ) { + self.keys = keys + self.limit = limit + self.maxchars = maxchars + } + private enum CodingKeys: String, CodingKey { + case keys + case limit + case maxchars = "maxChars" + } +} + +public struct SessionsResolveParams: Codable, Sendable { + public let key: String? + public let sessionid: String? + public let label: String? + public let agentid: String? + public let spawnedby: String? + public let includeglobal: Bool? + public let includeunknown: Bool? + + public init( + key: String?, + sessionid: String?, + label: String?, + agentid: String?, + spawnedby: String?, + includeglobal: Bool?, + includeunknown: Bool? + ) { + self.key = key + self.sessionid = sessionid + self.label = label + self.agentid = agentid + self.spawnedby = spawnedby + self.includeglobal = includeglobal + self.includeunknown = includeunknown + } + private enum CodingKeys: String, CodingKey { + case key + case sessionid = "sessionId" + case label + case agentid = "agentId" + case spawnedby = "spawnedBy" + case includeglobal = "includeGlobal" + case includeunknown = "includeUnknown" + } +} + +public struct SessionsPatchParams: Codable, Sendable { + public let key: String + public let label: AnyCodable? + public let thinkinglevel: AnyCodable? + public let verboselevel: AnyCodable? + public let reasoninglevel: AnyCodable? + public let responseusage: AnyCodable? + public let elevatedlevel: AnyCodable? + public let exechost: AnyCodable? + public let execsecurity: AnyCodable? + public let execask: AnyCodable? + public let execnode: AnyCodable? + public let model: AnyCodable? + public let spawnedby: AnyCodable? + public let sendpolicy: AnyCodable? + public let groupactivation: AnyCodable? + + public init( + key: String, + label: AnyCodable?, + thinkinglevel: AnyCodable?, + verboselevel: AnyCodable?, + reasoninglevel: AnyCodable?, + responseusage: AnyCodable?, + elevatedlevel: AnyCodable?, + exechost: AnyCodable?, + execsecurity: AnyCodable?, + execask: AnyCodable?, + execnode: AnyCodable?, + model: AnyCodable?, + spawnedby: AnyCodable?, + sendpolicy: AnyCodable?, + groupactivation: AnyCodable? + ) { + self.key = key + self.label = label + self.thinkinglevel = thinkinglevel + self.verboselevel = verboselevel + self.reasoninglevel = reasoninglevel + self.responseusage = responseusage + self.elevatedlevel = elevatedlevel + self.exechost = exechost + self.execsecurity = execsecurity + self.execask = execask + self.execnode = execnode + self.model = model + self.spawnedby = spawnedby + self.sendpolicy = sendpolicy + self.groupactivation = groupactivation + } + private enum CodingKeys: String, CodingKey { + case key + case label + case thinkinglevel = "thinkingLevel" + case verboselevel = "verboseLevel" + case reasoninglevel = "reasoningLevel" + case responseusage = "responseUsage" + case elevatedlevel = "elevatedLevel" + case exechost = "execHost" + case execsecurity = "execSecurity" + case execask = "execAsk" + case execnode = "execNode" + case model + case spawnedby = "spawnedBy" + case sendpolicy = "sendPolicy" + case groupactivation = "groupActivation" + } +} + +public struct SessionsResetParams: Codable, Sendable { + public let key: String + + public init( + key: String + ) { + self.key = key + } + private enum CodingKeys: String, CodingKey { + case key + } +} + +public struct SessionsDeleteParams: Codable, Sendable { + public let key: String + public let deletetranscript: Bool? + + public init( + key: String, + deletetranscript: Bool? + ) { + self.key = key + self.deletetranscript = deletetranscript + } + private enum CodingKeys: String, CodingKey { + case key + case deletetranscript = "deleteTranscript" + } +} + +public struct SessionsCompactParams: Codable, Sendable { + public let key: String + public let maxlines: Int? + + public init( + key: String, + maxlines: Int? + ) { + self.key = key + self.maxlines = maxlines + } + private enum CodingKeys: String, CodingKey { + case key + case maxlines = "maxLines" + } +} + +public struct ConfigGetParams: Codable, Sendable { +} + +public struct ConfigSetParams: Codable, Sendable { + public let raw: String + public let basehash: String? + + public init( + raw: String, + basehash: String? + ) { + self.raw = raw + self.basehash = basehash + } + private enum CodingKeys: String, CodingKey { + case raw + case basehash = "baseHash" + } +} + +public struct ConfigApplyParams: Codable, Sendable { + public let raw: String + public let basehash: String? + public let sessionkey: String? + public let note: String? + public let restartdelayms: Int? + + public init( + raw: String, + basehash: String?, + sessionkey: String?, + note: String?, + restartdelayms: Int? + ) { + self.raw = raw + self.basehash = basehash + self.sessionkey = sessionkey + self.note = note + self.restartdelayms = restartdelayms + } + private enum CodingKeys: String, CodingKey { + case raw + case basehash = "baseHash" + case sessionkey = "sessionKey" + case note + case restartdelayms = "restartDelayMs" + } +} + +public struct ConfigPatchParams: Codable, Sendable { + public let raw: String + public let basehash: String? + public let sessionkey: String? + public let note: String? + public let restartdelayms: Int? + + public init( + raw: String, + basehash: String?, + sessionkey: String?, + note: String?, + restartdelayms: Int? + ) { + self.raw = raw + self.basehash = basehash + self.sessionkey = sessionkey + self.note = note + self.restartdelayms = restartdelayms + } + private enum CodingKeys: String, CodingKey { + case raw + case basehash = "baseHash" + case sessionkey = "sessionKey" + case note + case restartdelayms = "restartDelayMs" + } +} + +public struct ConfigSchemaParams: Codable, Sendable { +} + +public struct ConfigSchemaResponse: Codable, Sendable { + public let schema: AnyCodable + public let uihints: [String: AnyCodable] + public let version: String + public let generatedat: String + + public init( + schema: AnyCodable, + uihints: [String: AnyCodable], + version: String, + generatedat: String + ) { + self.schema = schema + self.uihints = uihints + self.version = version + self.generatedat = generatedat + } + private enum CodingKeys: String, CodingKey { + case schema + case uihints = "uiHints" + case version + case generatedat = "generatedAt" + } +} + +public struct WizardStartParams: Codable, Sendable { + public let mode: AnyCodable? + public let workspace: String? + + public init( + mode: AnyCodable?, + workspace: String? + ) { + self.mode = mode + self.workspace = workspace + } + private enum CodingKeys: String, CodingKey { + case mode + case workspace + } +} + +public struct WizardNextParams: Codable, Sendable { + public let sessionid: String + public let answer: [String: AnyCodable]? + + public init( + sessionid: String, + answer: [String: AnyCodable]? + ) { + self.sessionid = sessionid + self.answer = answer + } + private enum CodingKeys: String, CodingKey { + case sessionid = "sessionId" + case answer + } +} + +public struct WizardCancelParams: Codable, Sendable { + public let sessionid: String + + public init( + sessionid: String + ) { + self.sessionid = sessionid + } + private enum CodingKeys: String, CodingKey { + case sessionid = "sessionId" + } +} + +public struct WizardStatusParams: Codable, Sendable { + public let sessionid: String + + public init( + sessionid: String + ) { + self.sessionid = sessionid + } + private enum CodingKeys: String, CodingKey { + case sessionid = "sessionId" + } +} + +public struct WizardStep: Codable, Sendable { + public let id: String + public let type: AnyCodable + public let title: String? + public let message: String? + public let options: [[String: AnyCodable]]? + public let initialvalue: AnyCodable? + public let placeholder: String? + public let sensitive: Bool? + public let executor: AnyCodable? + + public init( + id: String, + type: AnyCodable, + title: String?, + message: String?, + options: [[String: AnyCodable]]?, + initialvalue: AnyCodable?, + placeholder: String?, + sensitive: Bool?, + executor: AnyCodable? + ) { + self.id = id + self.type = type + self.title = title + self.message = message + self.options = options + self.initialvalue = initialvalue + self.placeholder = placeholder + self.sensitive = sensitive + self.executor = executor + } + private enum CodingKeys: String, CodingKey { + case id + case type + case title + case message + case options + case initialvalue = "initialValue" + case placeholder + case sensitive + case executor + } +} + +public struct WizardNextResult: Codable, Sendable { + public let done: Bool + public let step: [String: AnyCodable]? + public let status: AnyCodable? + public let error: String? + + public init( + done: Bool, + step: [String: AnyCodable]?, + status: AnyCodable?, + error: String? + ) { + self.done = done + self.step = step + self.status = status + self.error = error + } + private enum CodingKeys: String, CodingKey { + case done + case step + case status + case error + } +} + +public struct WizardStartResult: Codable, Sendable { + public let sessionid: String + public let done: Bool + public let step: [String: AnyCodable]? + public let status: AnyCodable? + public let error: String? + + public init( + sessionid: String, + done: Bool, + step: [String: AnyCodable]?, + status: AnyCodable?, + error: String? + ) { + self.sessionid = sessionid + self.done = done + self.step = step + self.status = status + self.error = error + } + private enum CodingKeys: String, CodingKey { + case sessionid = "sessionId" + case done + case step + case status + case error + } +} + +public struct WizardStatusResult: Codable, Sendable { + public let status: AnyCodable + public let error: String? + + public init( + status: AnyCodable, + error: String? + ) { + self.status = status + self.error = error + } + private enum CodingKeys: String, CodingKey { + case status + case error + } +} + +public struct TalkModeParams: Codable, Sendable { + public let enabled: Bool + public let phase: String? + + public init( + enabled: Bool, + phase: String? + ) { + self.enabled = enabled + self.phase = phase + } + private enum CodingKeys: String, CodingKey { + case enabled + case phase + } +} + +public struct ChannelsStatusParams: Codable, Sendable { + public let probe: Bool? + public let timeoutms: Int? + + public init( + probe: Bool?, + timeoutms: Int? + ) { + self.probe = probe + self.timeoutms = timeoutms + } + private enum CodingKeys: String, CodingKey { + case probe + case timeoutms = "timeoutMs" + } +} + +public struct ChannelsStatusResult: Codable, Sendable { + public let ts: Int + public let channelorder: [String] + public let channellabels: [String: AnyCodable] + public let channeldetaillabels: [String: AnyCodable]? + public let channelsystemimages: [String: AnyCodable]? + public let channelmeta: [[String: AnyCodable]]? + public let channels: [String: AnyCodable] + public let channelaccounts: [String: AnyCodable] + public let channeldefaultaccountid: [String: AnyCodable] + + public init( + ts: Int, + channelorder: [String], + channellabels: [String: AnyCodable], + channeldetaillabels: [String: AnyCodable]?, + channelsystemimages: [String: AnyCodable]?, + channelmeta: [[String: AnyCodable]]?, + channels: [String: AnyCodable], + channelaccounts: [String: AnyCodable], + channeldefaultaccountid: [String: AnyCodable] + ) { + self.ts = ts + self.channelorder = channelorder + self.channellabels = channellabels + self.channeldetaillabels = channeldetaillabels + self.channelsystemimages = channelsystemimages + self.channelmeta = channelmeta + self.channels = channels + self.channelaccounts = channelaccounts + self.channeldefaultaccountid = channeldefaultaccountid + } + private enum CodingKeys: String, CodingKey { + case ts + case channelorder = "channelOrder" + case channellabels = "channelLabels" + case channeldetaillabels = "channelDetailLabels" + case channelsystemimages = "channelSystemImages" + case channelmeta = "channelMeta" + case channels + case channelaccounts = "channelAccounts" + case channeldefaultaccountid = "channelDefaultAccountId" + } +} + +public struct ChannelsLogoutParams: Codable, Sendable { + public let channel: String + public let accountid: String? + + public init( + channel: String, + accountid: String? + ) { + self.channel = channel + self.accountid = accountid + } + private enum CodingKeys: String, CodingKey { + case channel + case accountid = "accountId" + } +} + +public struct WebLoginStartParams: Codable, Sendable { + public let force: Bool? + public let timeoutms: Int? + public let verbose: Bool? + public let accountid: String? + + public init( + force: Bool?, + timeoutms: Int?, + verbose: Bool?, + accountid: String? + ) { + self.force = force + self.timeoutms = timeoutms + self.verbose = verbose + self.accountid = accountid + } + private enum CodingKeys: String, CodingKey { + case force + case timeoutms = "timeoutMs" + case verbose + case accountid = "accountId" + } +} + +public struct WebLoginWaitParams: Codable, Sendable { + public let timeoutms: Int? + public let accountid: String? + + public init( + timeoutms: Int?, + accountid: String? + ) { + self.timeoutms = timeoutms + self.accountid = accountid + } + private enum CodingKeys: String, CodingKey { + case timeoutms = "timeoutMs" + case accountid = "accountId" + } +} + +public struct AgentSummary: Codable, Sendable { + public let id: String + public let name: String? + public let identity: [String: AnyCodable]? + + public init( + id: String, + name: String?, + identity: [String: AnyCodable]? + ) { + self.id = id + self.name = name + self.identity = identity + } + private enum CodingKeys: String, CodingKey { + case id + case name + case identity + } +} + +public struct AgentsListParams: Codable, Sendable { +} + +public struct AgentsListResult: Codable, Sendable { + public let defaultid: String + public let mainkey: String + public let scope: AnyCodable + public let agents: [AgentSummary] + + public init( + defaultid: String, + mainkey: String, + scope: AnyCodable, + agents: [AgentSummary] + ) { + self.defaultid = defaultid + self.mainkey = mainkey + self.scope = scope + self.agents = agents + } + private enum CodingKeys: String, CodingKey { + case defaultid = "defaultId" + case mainkey = "mainKey" + case scope + case agents + } +} + +public struct ModelChoice: Codable, Sendable { + public let id: String + public let name: String + public let provider: String + public let contextwindow: Int? + public let reasoning: Bool? + + public init( + id: String, + name: String, + provider: String, + contextwindow: Int?, + reasoning: Bool? + ) { + self.id = id + self.name = name + self.provider = provider + self.contextwindow = contextwindow + self.reasoning = reasoning + } + private enum CodingKeys: String, CodingKey { + case id + case name + case provider + case contextwindow = "contextWindow" + case reasoning + } +} + +public struct ModelsListParams: Codable, Sendable { +} + +public struct ModelsListResult: Codable, Sendable { + public let models: [ModelChoice] + + public init( + models: [ModelChoice] + ) { + self.models = models + } + private enum CodingKeys: String, CodingKey { + case models + } +} + +public struct SkillsStatusParams: Codable, Sendable { +} + +public struct SkillsBinsParams: Codable, Sendable { +} + +public struct SkillsBinsResult: Codable, Sendable { + public let bins: [String] + + public init( + bins: [String] + ) { + self.bins = bins + } + private enum CodingKeys: String, CodingKey { + case bins + } +} + +public struct SkillsInstallParams: Codable, Sendable { + public let name: String + public let installid: String + public let timeoutms: Int? + + public init( + name: String, + installid: String, + timeoutms: Int? + ) { + self.name = name + self.installid = installid + self.timeoutms = timeoutms + } + private enum CodingKeys: String, CodingKey { + case name + case installid = "installId" + case timeoutms = "timeoutMs" + } +} + +public struct SkillsUpdateParams: Codable, Sendable { + public let skillkey: String + public let enabled: Bool? + public let apikey: String? + public let env: [String: AnyCodable]? + + public init( + skillkey: String, + enabled: Bool?, + apikey: String?, + env: [String: AnyCodable]? + ) { + self.skillkey = skillkey + self.enabled = enabled + self.apikey = apikey + self.env = env + } + private enum CodingKeys: String, CodingKey { + case skillkey = "skillKey" + case enabled + case apikey = "apiKey" + case env + } +} + +public struct CronJob: Codable, Sendable { + public let id: String + public let agentid: String? + public let name: String + public let description: String? + public let enabled: Bool + public let deleteafterrun: Bool? + public let createdatms: Int + public let updatedatms: Int + public let schedule: AnyCodable + public let sessiontarget: AnyCodable + public let wakemode: AnyCodable + public let payload: AnyCodable + public let isolation: [String: AnyCodable]? + public let state: [String: AnyCodable] + + public init( + id: String, + agentid: String?, + name: String, + description: String?, + enabled: Bool, + deleteafterrun: Bool?, + createdatms: Int, + updatedatms: Int, + schedule: AnyCodable, + sessiontarget: AnyCodable, + wakemode: AnyCodable, + payload: AnyCodable, + isolation: [String: AnyCodable]?, + state: [String: AnyCodable] + ) { + self.id = id + self.agentid = agentid + self.name = name + self.description = description + self.enabled = enabled + self.deleteafterrun = deleteafterrun + self.createdatms = createdatms + self.updatedatms = updatedatms + self.schedule = schedule + self.sessiontarget = sessiontarget + self.wakemode = wakemode + self.payload = payload + self.isolation = isolation + self.state = state + } + private enum CodingKeys: String, CodingKey { + case id + case agentid = "agentId" + case name + case description + case enabled + case deleteafterrun = "deleteAfterRun" + case createdatms = "createdAtMs" + case updatedatms = "updatedAtMs" + case schedule + case sessiontarget = "sessionTarget" + case wakemode = "wakeMode" + case payload + case isolation + case state + } +} + +public struct CronListParams: Codable, Sendable { + public let includedisabled: Bool? + + public init( + includedisabled: Bool? + ) { + self.includedisabled = includedisabled + } + private enum CodingKeys: String, CodingKey { + case includedisabled = "includeDisabled" + } +} + +public struct CronStatusParams: Codable, Sendable { +} + +public struct CronAddParams: Codable, Sendable { + public let name: String + public let agentid: AnyCodable? + public let description: String? + public let enabled: Bool? + public let deleteafterrun: Bool? + public let schedule: AnyCodable + public let sessiontarget: AnyCodable + public let wakemode: AnyCodable + public let payload: AnyCodable + public let isolation: [String: AnyCodable]? + + public init( + name: String, + agentid: AnyCodable?, + description: String?, + enabled: Bool?, + deleteafterrun: Bool?, + schedule: AnyCodable, + sessiontarget: AnyCodable, + wakemode: AnyCodable, + payload: AnyCodable, + isolation: [String: AnyCodable]? + ) { + self.name = name + self.agentid = agentid + self.description = description + self.enabled = enabled + self.deleteafterrun = deleteafterrun + self.schedule = schedule + self.sessiontarget = sessiontarget + self.wakemode = wakemode + self.payload = payload + self.isolation = isolation + } + private enum CodingKeys: String, CodingKey { + case name + case agentid = "agentId" + case description + case enabled + case deleteafterrun = "deleteAfterRun" + case schedule + case sessiontarget = "sessionTarget" + case wakemode = "wakeMode" + case payload + case isolation + } +} + +public struct CronRunLogEntry: Codable, Sendable { + public let ts: Int + public let jobid: String + public let action: String + public let status: AnyCodable? + public let error: String? + public let summary: String? + public let runatms: Int? + public let durationms: Int? + public let nextrunatms: Int? + + public init( + ts: Int, + jobid: String, + action: String, + status: AnyCodable?, + error: String?, + summary: String?, + runatms: Int?, + durationms: Int?, + nextrunatms: Int? + ) { + self.ts = ts + self.jobid = jobid + self.action = action + self.status = status + self.error = error + self.summary = summary + self.runatms = runatms + self.durationms = durationms + self.nextrunatms = nextrunatms + } + private enum CodingKeys: String, CodingKey { + case ts + case jobid = "jobId" + case action + case status + case error + case summary + case runatms = "runAtMs" + case durationms = "durationMs" + case nextrunatms = "nextRunAtMs" + } +} + +public struct LogsTailParams: Codable, Sendable { + public let cursor: Int? + public let limit: Int? + public let maxbytes: Int? + + public init( + cursor: Int?, + limit: Int?, + maxbytes: Int? + ) { + self.cursor = cursor + self.limit = limit + self.maxbytes = maxbytes + } + private enum CodingKeys: String, CodingKey { + case cursor + case limit + case maxbytes = "maxBytes" + } +} + +public struct LogsTailResult: Codable, Sendable { + public let file: String + public let cursor: Int + public let size: Int + public let lines: [String] + public let truncated: Bool? + public let reset: Bool? + + public init( + file: String, + cursor: Int, + size: Int, + lines: [String], + truncated: Bool?, + reset: Bool? + ) { + self.file = file + self.cursor = cursor + self.size = size + self.lines = lines + self.truncated = truncated + self.reset = reset + } + private enum CodingKeys: String, CodingKey { + case file + case cursor + case size + case lines + case truncated + case reset + } +} + +public struct ExecApprovalsGetParams: Codable, Sendable { +} + +public struct ExecApprovalsSetParams: Codable, Sendable { + public let file: [String: AnyCodable] + public let basehash: String? + + public init( + file: [String: AnyCodable], + basehash: String? + ) { + self.file = file + self.basehash = basehash + } + private enum CodingKeys: String, CodingKey { + case file + case basehash = "baseHash" + } +} + +public struct ExecApprovalsNodeGetParams: Codable, Sendable { + public let nodeid: String + + public init( + nodeid: String + ) { + self.nodeid = nodeid + } + private enum CodingKeys: String, CodingKey { + case nodeid = "nodeId" + } +} + +public struct ExecApprovalsNodeSetParams: Codable, Sendable { + public let nodeid: String + public let file: [String: AnyCodable] + public let basehash: String? + + public init( + nodeid: String, + file: [String: AnyCodable], + basehash: String? + ) { + self.nodeid = nodeid + self.file = file + self.basehash = basehash + } + private enum CodingKeys: String, CodingKey { + case nodeid = "nodeId" + case file + case basehash = "baseHash" + } +} + +public struct ExecApprovalsSnapshot: Codable, Sendable { + public let path: String + public let exists: Bool + public let hash: String + public let file: [String: AnyCodable] + + public init( + path: String, + exists: Bool, + hash: String, + file: [String: AnyCodable] + ) { + self.path = path + self.exists = exists + self.hash = hash + self.file = file + } + private enum CodingKeys: String, CodingKey { + case path + case exists + case hash + case file + } +} + +public struct ExecApprovalRequestParams: Codable, Sendable { + public let id: String? + public let command: String + public let cwd: AnyCodable? + public let host: AnyCodable? + public let security: AnyCodable? + public let ask: AnyCodable? + public let agentid: AnyCodable? + public let resolvedpath: AnyCodable? + public let sessionkey: AnyCodable? + public let timeoutms: Int? + + public init( + id: String?, + command: String, + cwd: AnyCodable?, + host: AnyCodable?, + security: AnyCodable?, + ask: AnyCodable?, + agentid: AnyCodable?, + resolvedpath: AnyCodable?, + sessionkey: AnyCodable?, + timeoutms: Int? + ) { + self.id = id + self.command = command + self.cwd = cwd + self.host = host + self.security = security + self.ask = ask + self.agentid = agentid + self.resolvedpath = resolvedpath + self.sessionkey = sessionkey + self.timeoutms = timeoutms + } + private enum CodingKeys: String, CodingKey { + case id + case command + case cwd + case host + case security + case ask + case agentid = "agentId" + case resolvedpath = "resolvedPath" + case sessionkey = "sessionKey" + case timeoutms = "timeoutMs" + } +} + +public struct ExecApprovalResolveParams: Codable, Sendable { + public let id: String + public let decision: String + + public init( + id: String, + decision: String + ) { + self.id = id + self.decision = decision + } + private enum CodingKeys: String, CodingKey { + case id + case decision + } +} + +public struct DevicePairListParams: Codable, Sendable { +} + +public struct DevicePairApproveParams: Codable, Sendable { + public let requestid: String + + public init( + requestid: String + ) { + self.requestid = requestid + } + private enum CodingKeys: String, CodingKey { + case requestid = "requestId" + } +} + +public struct DevicePairRejectParams: Codable, Sendable { + public let requestid: String + + public init( + requestid: String + ) { + self.requestid = requestid + } + private enum CodingKeys: String, CodingKey { + case requestid = "requestId" + } +} + +public struct DeviceTokenRotateParams: Codable, Sendable { + public let deviceid: String + public let role: String + public let scopes: [String]? + + public init( + deviceid: String, + role: String, + scopes: [String]? + ) { + self.deviceid = deviceid + self.role = role + self.scopes = scopes + } + private enum CodingKeys: String, CodingKey { + case deviceid = "deviceId" + case role + case scopes + } +} + +public struct DeviceTokenRevokeParams: Codable, Sendable { + public let deviceid: String + public let role: String + + public init( + deviceid: String, + role: String + ) { + self.deviceid = deviceid + self.role = role + } + private enum CodingKeys: String, CodingKey { + case deviceid = "deviceId" + case role + } +} + +public struct DevicePairRequestedEvent: Codable, Sendable { + public let requestid: String + public let deviceid: String + public let publickey: String + public let displayname: String? + public let platform: String? + public let clientid: String? + public let clientmode: String? + public let role: String? + public let roles: [String]? + public let scopes: [String]? + public let remoteip: String? + public let silent: Bool? + public let isrepair: Bool? + public let ts: Int + + public init( + requestid: String, + deviceid: String, + publickey: String, + displayname: String?, + platform: String?, + clientid: String?, + clientmode: String?, + role: String?, + roles: [String]?, + scopes: [String]?, + remoteip: String?, + silent: Bool?, + isrepair: Bool?, + ts: Int + ) { + self.requestid = requestid + self.deviceid = deviceid + self.publickey = publickey + self.displayname = displayname + self.platform = platform + self.clientid = clientid + self.clientmode = clientmode + self.role = role + self.roles = roles + self.scopes = scopes + self.remoteip = remoteip + self.silent = silent + self.isrepair = isrepair + self.ts = ts + } + private enum CodingKeys: String, CodingKey { + case requestid = "requestId" + case deviceid = "deviceId" + case publickey = "publicKey" + case displayname = "displayName" + case platform + case clientid = "clientId" + case clientmode = "clientMode" + case role + case roles + case scopes + case remoteip = "remoteIp" + case silent + case isrepair = "isRepair" + case ts + } +} + +public struct DevicePairResolvedEvent: Codable, Sendable { + public let requestid: String + public let deviceid: String + public let decision: String + public let ts: Int + + public init( + requestid: String, + deviceid: String, + decision: String, + ts: Int + ) { + self.requestid = requestid + self.deviceid = deviceid + self.decision = decision + self.ts = ts + } + private enum CodingKeys: String, CodingKey { + case requestid = "requestId" + case deviceid = "deviceId" + case decision + case ts + } +} + +public struct ChatHistoryParams: Codable, Sendable { + public let sessionkey: String + public let limit: Int? + + public init( + sessionkey: String, + limit: Int? + ) { + self.sessionkey = sessionkey + self.limit = limit + } + private enum CodingKeys: String, CodingKey { + case sessionkey = "sessionKey" + case limit + } +} + +public struct ChatSendParams: Codable, Sendable { + public let sessionkey: String + public let message: String + public let thinking: String? + public let deliver: Bool? + public let attachments: [AnyCodable]? + public let timeoutms: Int? + public let idempotencykey: String + + public init( + sessionkey: String, + message: String, + thinking: String?, + deliver: Bool?, + attachments: [AnyCodable]?, + timeoutms: Int?, + idempotencykey: String + ) { + self.sessionkey = sessionkey + self.message = message + self.thinking = thinking + self.deliver = deliver + self.attachments = attachments + self.timeoutms = timeoutms + self.idempotencykey = idempotencykey + } + private enum CodingKeys: String, CodingKey { + case sessionkey = "sessionKey" + case message + case thinking + case deliver + case attachments + case timeoutms = "timeoutMs" + case idempotencykey = "idempotencyKey" + } +} + +public struct ChatAbortParams: Codable, Sendable { + public let sessionkey: String + public let runid: String? + + public init( + sessionkey: String, + runid: String? + ) { + self.sessionkey = sessionkey + self.runid = runid + } + private enum CodingKeys: String, CodingKey { + case sessionkey = "sessionKey" + case runid = "runId" + } +} + +public struct ChatInjectParams: Codable, Sendable { + public let sessionkey: String + public let message: String + public let label: String? + + public init( + sessionkey: String, + message: String, + label: String? + ) { + self.sessionkey = sessionkey + self.message = message + self.label = label + } + private enum CodingKeys: String, CodingKey { + case sessionkey = "sessionKey" + case message + case label + } +} + +public struct ChatEvent: Codable, Sendable { + public let runid: String + public let sessionkey: String + public let seq: Int + public let state: AnyCodable + public let message: AnyCodable? + public let errormessage: String? + public let usage: AnyCodable? + public let stopreason: String? + + public init( + runid: String, + sessionkey: String, + seq: Int, + state: AnyCodable, + message: AnyCodable?, + errormessage: String?, + usage: AnyCodable?, + stopreason: String? + ) { + self.runid = runid + self.sessionkey = sessionkey + self.seq = seq + self.state = state + self.message = message + self.errormessage = errormessage + self.usage = usage + self.stopreason = stopreason + } + private enum CodingKeys: String, CodingKey { + case runid = "runId" + case sessionkey = "sessionKey" + case seq + case state + case message + case errormessage = "errorMessage" + case usage + case stopreason = "stopReason" + } +} + +public struct UpdateRunParams: Codable, Sendable { + public let sessionkey: String? + public let note: String? + public let restartdelayms: Int? + public let timeoutms: Int? + + public init( + sessionkey: String?, + note: String?, + restartdelayms: Int?, + timeoutms: Int? + ) { + self.sessionkey = sessionkey + self.note = note + self.restartdelayms = restartdelayms + self.timeoutms = timeoutms + } + private enum CodingKeys: String, CodingKey { + case sessionkey = "sessionKey" + case note + case restartdelayms = "restartDelayMs" + case timeoutms = "timeoutMs" + } +} + +public struct TickEvent: Codable, Sendable { + public let ts: Int + + public init( + ts: Int + ) { + self.ts = ts + } + private enum CodingKeys: String, CodingKey { + case ts + } +} + +public struct ShutdownEvent: Codable, Sendable { + public let reason: String + public let restartexpectedms: Int? + + public init( + reason: String, + restartexpectedms: Int? + ) { + self.reason = reason + self.restartexpectedms = restartexpectedms + } + private enum CodingKeys: String, CodingKey { + case reason + case restartexpectedms = "restartExpectedMs" + } +} + +public enum GatewayFrame: Codable, Sendable { + case req(RequestFrame) + case res(ResponseFrame) + case event(EventFrame) + case unknown(type: String, raw: [String: AnyCodable]) + + private enum CodingKeys: String, CodingKey { + case type + } + + public init(from decoder: Decoder) throws { + let typeContainer = try decoder.container(keyedBy: CodingKeys.self) + let type = try typeContainer.decode(String.self, forKey: .type) + switch type { + case "req": + self = .req(try RequestFrame(from: decoder)) + case "res": + self = .res(try ResponseFrame(from: decoder)) + case "event": + self = .event(try EventFrame(from: decoder)) + default: + let container = try decoder.singleValueContainer() + let raw = try container.decode([String: AnyCodable].self) + self = .unknown(type: type, raw: raw) + } + } + + public func encode(to encoder: Encoder) throws { + switch self { + case .req(let v): try v.encode(to: encoder) + case .res(let v): try v.encode(to: encoder) + case .event(let v): try v.encode(to: encoder) + case .unknown(_, let raw): + var container = encoder.singleValueContainer() + try container.encode(raw) + } + } + +} diff --git a/apps/shared/OpenClawKit/Sources/OpenClawProtocol/WizardHelpers.swift b/apps/shared/OpenClawKit/Sources/OpenClawProtocol/WizardHelpers.swift new file mode 100644 index 0000000000000000000000000000000000000000..d410914bfa57151576ae206f1305d1c88289da62 --- /dev/null +++ b/apps/shared/OpenClawKit/Sources/OpenClawProtocol/WizardHelpers.swift @@ -0,0 +1,106 @@ +import Foundation + +public struct WizardOption: Sendable { + public let value: AnyCodable? + public let label: String + public let hint: String? + + public init(value: AnyCodable?, label: String, hint: String?) { + self.value = value + self.label = label + self.hint = hint + } +} + +public func decodeWizardStep(_ raw: [String: AnyCodable]?) -> WizardStep? { + guard let raw else { return nil } + do { + let data = try JSONEncoder().encode(raw) + return try JSONDecoder().decode(WizardStep.self, from: data) + } catch { + return nil + } +} + +public func parseWizardOptions(_ raw: [[String: AnyCodable]]?) -> [WizardOption] { + guard let raw else { return [] } + return raw.map { entry in + let value = entry["value"] + let label = (entry["label"]?.value as? String) ?? "" + let hint = entry["hint"]?.value as? String + return WizardOption(value: value, label: label, hint: hint) + } +} + +public func wizardStatusString(_ value: AnyCodable?) -> String? { + (value?.value as? String)?.trimmingCharacters(in: .whitespacesAndNewlines).lowercased() +} + +public func wizardStepType(_ step: WizardStep) -> String { + (step.type.value as? String) ?? "" +} + +public func anyCodableString(_ value: AnyCodable?) -> String { + switch value?.value { + case let string as String: + string + case let int as Int: + String(int) + case let double as Double: + String(double) + case let bool as Bool: + bool ? "true" : "false" + default: + "" + } +} + +public func anyCodableBool(_ value: AnyCodable?) -> Bool { + switch value?.value { + case let bool as Bool: + return bool + case let int as Int: + return int != 0 + case let double as Double: + return double != 0 + case let string as String: + let trimmed = string.trimmingCharacters(in: .whitespacesAndNewlines).lowercased() + return trimmed == "true" || trimmed == "1" || trimmed == "yes" + default: + return false + } +} + +public func anyCodableArray(_ value: AnyCodable?) -> [AnyCodable] { + switch value?.value { + case let arr as [AnyCodable]: + return arr + case let arr as [Any]: + return arr.map { AnyCodable($0) } + default: + return [] + } +} + +public func anyCodableEqual(_ lhs: AnyCodable?, _ rhs: AnyCodable?) -> Bool { + switch (lhs?.value, rhs?.value) { + case let (l as String, r as String): + l == r + case let (l as Int, r as Int): + l == r + case let (l as Double, r as Double): + l == r + case let (l as Bool, r as Bool): + l == r + case let (l as String, r as Int): + l == String(r) + case let (l as Int, r as String): + String(l) == r + case let (l as String, r as Double): + l == String(r) + case let (l as Double, r as String): + String(l) == r + default: + false + } +} diff --git a/apps/shared/OpenClawKit/Tests/OpenClawKitTests/AssistantTextParserTests.swift b/apps/shared/OpenClawKit/Tests/OpenClawKitTests/AssistantTextParserTests.swift new file mode 100644 index 0000000000000000000000000000000000000000..5f36bb9c267261bcc3d6c18cbb822b44327ee070 --- /dev/null +++ b/apps/shared/OpenClawKit/Tests/OpenClawKitTests/AssistantTextParserTests.swift @@ -0,0 +1,37 @@ +import Testing +@testable import OpenClawChatUI + +@Suite struct AssistantTextParserTests { + @Test func splitsThinkAndFinalSegments() { + let segments = AssistantTextParser.segments( + from: "internal\n\nHello there") + + #expect(segments.count == 2) + #expect(segments[0].kind == .thinking) + #expect(segments[0].text == "internal") + #expect(segments[1].kind == .response) + #expect(segments[1].text == "Hello there") + } + + @Test func keepsTextWithoutTags() { + let segments = AssistantTextParser.segments(from: "Just text.") + + #expect(segments.count == 1) + #expect(segments[0].kind == .response) + #expect(segments[0].text == "Just text.") + } + + @Test func ignoresThinkingLikeTags() { + let raw = "example\nKeep this." + let segments = AssistantTextParser.segments(from: raw) + + #expect(segments.count == 1) + #expect(segments[0].kind == .response) + #expect(segments[0].text == raw.trimmingCharacters(in: .whitespacesAndNewlines)) + } + + @Test func dropsEmptyTaggedContent() { + let segments = AssistantTextParser.segments(from: "") + #expect(segments.isEmpty) + } +} diff --git a/apps/shared/OpenClawKit/Tests/OpenClawKitTests/BonjourEscapesTests.swift b/apps/shared/OpenClawKit/Tests/OpenClawKitTests/BonjourEscapesTests.swift new file mode 100644 index 0000000000000000000000000000000000000000..a7fa1438d3cb3e9caf20fb96c7893c8abf99c551 --- /dev/null +++ b/apps/shared/OpenClawKit/Tests/OpenClawKitTests/BonjourEscapesTests.swift @@ -0,0 +1,26 @@ +import OpenClawKit +import Testing + +@Suite struct BonjourEscapesTests { + @Test func decodePassThrough() { + #expect(BonjourEscapes.decode("hello") == "hello") + #expect(BonjourEscapes.decode("") == "") + } + + @Test func decodeSpaces() { + #expect(BonjourEscapes.decode("OpenClaw\\032Gateway") == "OpenClaw Gateway") + } + + @Test func decodeMultipleEscapes() { + #expect(BonjourEscapes.decode("A\\038B\\047C\\032D") == "A&B/C D") + } + + @Test func decodeIgnoresInvalidEscapeSequences() { + #expect(BonjourEscapes.decode("Hello\\03World") == "Hello\\03World") + #expect(BonjourEscapes.decode("Hello\\XYZWorld") == "Hello\\XYZWorld") + } + + @Test func decodeUsesDecimalUnicodeScalarValue() { + #expect(BonjourEscapes.decode("Hello\\065World") == "HelloAWorld") + } +} diff --git a/apps/shared/OpenClawKit/Tests/OpenClawKitTests/CanvasA2UIActionTests.swift b/apps/shared/OpenClawKit/Tests/OpenClawKitTests/CanvasA2UIActionTests.swift new file mode 100644 index 0000000000000000000000000000000000000000..f6070f6de8d9c30a94c5f917960992dff972a955 --- /dev/null +++ b/apps/shared/OpenClawKit/Tests/OpenClawKitTests/CanvasA2UIActionTests.swift @@ -0,0 +1,36 @@ +import OpenClawKit +import Foundation +import Testing + +@Suite struct CanvasA2UIActionTests { + @Test func sanitizeTagValueIsStable() { + #expect(OpenClawCanvasA2UIAction.sanitizeTagValue("Hello World!") == "Hello_World_") + #expect(OpenClawCanvasA2UIAction.sanitizeTagValue(" ") == "-") + #expect(OpenClawCanvasA2UIAction.sanitizeTagValue("macOS 26.2") == "macOS_26.2") + } + + @Test func extractActionNameAcceptsNameOrAction() { + #expect(OpenClawCanvasA2UIAction.extractActionName(["name": "Hello"]) == "Hello") + #expect(OpenClawCanvasA2UIAction.extractActionName(["action": "Wave"]) == "Wave") + #expect(OpenClawCanvasA2UIAction.extractActionName(["name": " ", "action": "Fallback"]) == "Fallback") + #expect(OpenClawCanvasA2UIAction.extractActionName(["action": " "]) == nil) + } + + @Test func formatAgentMessageIsTokenEfficientAndUnambiguous() { + let messageContext = OpenClawCanvasA2UIAction.AgentMessageContext( + actionName: "Get Weather", + session: .init(key: "main", surfaceId: "main"), + component: .init(id: "btnWeather", host: "Peter’s iPad", instanceId: "ipad16,6"), + contextJSON: "{\"city\":\"Vienna\"}") + let msg = OpenClawCanvasA2UIAction.formatAgentMessage(messageContext) + + #expect(msg.contains("CANVAS_A2UI ")) + #expect(msg.contains("action=Get_Weather")) + #expect(msg.contains("session=main")) + #expect(msg.contains("surface=main")) + #expect(msg.contains("component=btnWeather")) + #expect(msg.contains("host=Peter_s_iPad")) + #expect(msg.contains("instance=ipad16_6 ctx={\"city\":\"Vienna\"}")) + #expect(msg.hasSuffix(" default=update_canvas")) + } +} diff --git a/apps/shared/OpenClawKit/Tests/OpenClawKitTests/CanvasA2UITests.swift b/apps/shared/OpenClawKit/Tests/OpenClawKitTests/CanvasA2UITests.swift new file mode 100644 index 0000000000000000000000000000000000000000..4c420cc944c03c03ae995d8c05451b6dde6f60f9 --- /dev/null +++ b/apps/shared/OpenClawKit/Tests/OpenClawKitTests/CanvasA2UITests.swift @@ -0,0 +1,42 @@ +import OpenClawKit +import Testing + +@Suite struct CanvasA2UITests { + @Test func commandStringsAreStable() { + #expect(OpenClawCanvasA2UICommand.push.rawValue == "canvas.a2ui.push") + #expect(OpenClawCanvasA2UICommand.pushJSONL.rawValue == "canvas.a2ui.pushJSONL") + #expect(OpenClawCanvasA2UICommand.reset.rawValue == "canvas.a2ui.reset") + } + + @Test func jsonlDecodesAndValidatesV0_8() throws { + let jsonl = """ + {"beginRendering":{"surfaceId":"main","timestamp":1}} + {"surfaceUpdate":{"surfaceId":"main","ops":[]}} + {"dataModelUpdate":{"dataModel":{"title":"Hello"}}} + {"deleteSurface":{"surfaceId":"main"}} + """ + + let messages = try OpenClawCanvasA2UIJSONL.decodeMessagesFromJSONL(jsonl) + #expect(messages.count == 4) + } + + @Test func jsonlRejectsV0_9CreateSurface() { + let jsonl = """ + {"createSurface":{"surfaceId":"main"}} + """ + + #expect(throws: Error.self) { + _ = try OpenClawCanvasA2UIJSONL.decodeMessagesFromJSONL(jsonl) + } + } + + @Test func jsonlRejectsUnknownShape() { + let jsonl = """ + {"wat":{"nope":1}} + """ + + #expect(throws: Error.self) { + _ = try OpenClawCanvasA2UIJSONL.decodeMessagesFromJSONL(jsonl) + } + } +} diff --git a/apps/shared/OpenClawKit/Tests/OpenClawKitTests/CanvasSnapshotFormatTests.swift b/apps/shared/OpenClawKit/Tests/OpenClawKitTests/CanvasSnapshotFormatTests.swift new file mode 100644 index 0000000000000000000000000000000000000000..ab49a4f465fe113b0d2b2671ab3d044f54bea736 --- /dev/null +++ b/apps/shared/OpenClawKit/Tests/OpenClawKitTests/CanvasSnapshotFormatTests.swift @@ -0,0 +1,15 @@ +import OpenClawKit +import Foundation +import Testing + +@Suite struct CanvasSnapshotFormatTests { + @Test func acceptsJpgAlias() throws { + struct Wrapper: Codable { + var format: OpenClawCanvasSnapshotFormat + } + + let data = try #require("{\"format\":\"jpg\"}".data(using: .utf8)) + let decoded = try JSONDecoder().decode(Wrapper.self, from: data) + #expect(decoded.format == .jpeg) + } +} diff --git a/apps/shared/OpenClawKit/Tests/OpenClawKitTests/ChatMarkdownPreprocessorTests.swift b/apps/shared/OpenClawKit/Tests/OpenClawKitTests/ChatMarkdownPreprocessorTests.swift new file mode 100644 index 0000000000000000000000000000000000000000..808f74af64fa539d9141a0f5d746444fc8150832 --- /dev/null +++ b/apps/shared/OpenClawKit/Tests/OpenClawKitTests/ChatMarkdownPreprocessorTests.swift @@ -0,0 +1,20 @@ +import Testing +@testable import OpenClawChatUI + +@Suite("ChatMarkdownPreprocessor") +struct ChatMarkdownPreprocessorTests { + @Test func extractsDataURLImages() { + let base64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVQIHWP4////GQAJ+wP/2hN8NwAAAABJRU5ErkJggg==" + let markdown = """ + Hello + + ![Pixel](data:image/png;base64,\(base64)) + """ + + let result = ChatMarkdownPreprocessor.preprocess(markdown: markdown) + + #expect(result.cleaned == "Hello") + #expect(result.images.count == 1) + #expect(result.images.first?.image != nil) + } +} diff --git a/apps/shared/OpenClawKit/Tests/OpenClawKitTests/ChatThemeTests.swift b/apps/shared/OpenClawKit/Tests/OpenClawKitTests/ChatThemeTests.swift new file mode 100644 index 0000000000000000000000000000000000000000..2c7a5fff1eed7f127cd13f762da4fc43b4403660 --- /dev/null +++ b/apps/shared/OpenClawKit/Tests/OpenClawKitTests/ChatThemeTests.swift @@ -0,0 +1,29 @@ +import Foundation +import Testing +@testable import OpenClawChatUI + +#if os(macOS) +import AppKit +#endif + +#if os(macOS) +private func luminance(_ color: NSColor) throws -> CGFloat { + let rgb = try #require(color.usingColorSpace(.deviceRGB)) + return 0.2126 * rgb.redComponent + 0.7152 * rgb.greenComponent + 0.0722 * rgb.blueComponent +} +#endif + +@Suite struct ChatThemeTests { + @Test func assistantBubbleResolvesForLightAndDark() throws { + #if os(macOS) + let lightAppearance = try #require(NSAppearance(named: .aqua)) + let darkAppearance = try #require(NSAppearance(named: .darkAqua)) + + let lightResolved = OpenClawChatTheme.resolvedAssistantBubbleColor(for: lightAppearance) + let darkResolved = OpenClawChatTheme.resolvedAssistantBubbleColor(for: darkAppearance) + #expect(try luminance(lightResolved) > luminance(darkResolved)) + #else + #expect(Bool(true)) + #endif + } +} diff --git a/apps/shared/OpenClawKit/Tests/OpenClawKitTests/ChatViewModelTests.swift b/apps/shared/OpenClawKit/Tests/OpenClawKitTests/ChatViewModelTests.swift new file mode 100644 index 0000000000000000000000000000000000000000..3babe8b9a30c771381a3fbf74d910712f6ba735b --- /dev/null +++ b/apps/shared/OpenClawKit/Tests/OpenClawKitTests/ChatViewModelTests.swift @@ -0,0 +1,502 @@ +import OpenClawKit +import Foundation +import Testing +@testable import OpenClawChatUI + +private struct TimeoutError: Error, CustomStringConvertible { + let label: String + var description: String { "Timeout waiting for: \(self.label)" } +} + +private func waitUntil( + _ label: String, + timeoutSeconds: Double = 2.0, + pollMs: UInt64 = 10, + _ condition: @escaping @Sendable () async -> Bool) async throws +{ + let deadline = Date().addingTimeInterval(timeoutSeconds) + while Date() < deadline { + if await condition() { + return + } + try await Task.sleep(nanoseconds: pollMs * 1_000_000) + } + throw TimeoutError(label: label) +} + +private actor TestChatTransportState { + var historyCallCount: Int = 0 + var sessionsCallCount: Int = 0 + var sentRunIds: [String] = [] + var abortedRunIds: [String] = [] +} + +private final class TestChatTransport: @unchecked Sendable, OpenClawChatTransport { + private let state = TestChatTransportState() + private let historyResponses: [OpenClawChatHistoryPayload] + private let sessionsResponses: [OpenClawChatSessionsListResponse] + + private let stream: AsyncStream + private let continuation: AsyncStream.Continuation + + init( + historyResponses: [OpenClawChatHistoryPayload], + sessionsResponses: [OpenClawChatSessionsListResponse] = []) + { + self.historyResponses = historyResponses + self.sessionsResponses = sessionsResponses + var cont: AsyncStream.Continuation! + self.stream = AsyncStream { c in + cont = c + } + self.continuation = cont + } + + func events() -> AsyncStream { + self.stream + } + + func setActiveSessionKey(_: String) async throws {} + + func requestHistory(sessionKey: String) async throws -> OpenClawChatHistoryPayload { + let idx = await self.state.historyCallCount + await self.state.setHistoryCallCount(idx + 1) + if idx < self.historyResponses.count { + return self.historyResponses[idx] + } + return self.historyResponses.last ?? OpenClawChatHistoryPayload( + sessionKey: sessionKey, + sessionId: nil, + messages: [], + thinkingLevel: "off") + } + + func sendMessage( + sessionKey _: String, + message _: String, + thinking _: String, + idempotencyKey: String, + attachments _: [OpenClawChatAttachmentPayload]) async throws -> OpenClawChatSendResponse + { + await self.state.sentRunIdsAppend(idempotencyKey) + return OpenClawChatSendResponse(runId: idempotencyKey, status: "ok") + } + + func abortRun(sessionKey _: String, runId: String) async throws { + await self.state.abortedRunIdsAppend(runId) + } + + func listSessions(limit _: Int?) async throws -> OpenClawChatSessionsListResponse { + let idx = await self.state.sessionsCallCount + await self.state.setSessionsCallCount(idx + 1) + if idx < self.sessionsResponses.count { + return self.sessionsResponses[idx] + } + return self.sessionsResponses.last ?? OpenClawChatSessionsListResponse( + ts: nil, + path: nil, + count: 0, + defaults: nil, + sessions: []) + } + + func requestHealth(timeoutMs _: Int) async throws -> Bool { + true + } + + func emit(_ evt: OpenClawChatTransportEvent) { + self.continuation.yield(evt) + } + + func lastSentRunId() async -> String? { + let ids = await self.state.sentRunIds + return ids.last + } + + func abortedRunIds() async -> [String] { + await self.state.abortedRunIds + } +} + +extension TestChatTransportState { + fileprivate func setHistoryCallCount(_ v: Int) { + self.historyCallCount = v + } + + fileprivate func setSessionsCallCount(_ v: Int) { + self.sessionsCallCount = v + } + + fileprivate func sentRunIdsAppend(_ v: String) { + self.sentRunIds.append(v) + } + + fileprivate func abortedRunIdsAppend(_ v: String) { + self.abortedRunIds.append(v) + } +} + +@Suite struct ChatViewModelTests { + @Test func streamsAssistantAndClearsOnFinal() async throws { + let sessionId = "sess-main" + let history1 = OpenClawChatHistoryPayload( + sessionKey: "main", + sessionId: sessionId, + messages: [], + thinkingLevel: "off") + let history2 = OpenClawChatHistoryPayload( + sessionKey: "main", + sessionId: sessionId, + messages: [ + AnyCodable([ + "role": "assistant", + "content": [["type": "text", "text": "final answer"]], + "timestamp": Date().timeIntervalSince1970 * 1000, + ]), + ], + thinkingLevel: "off") + + let transport = TestChatTransport(historyResponses: [history1, history2]) + let vm = await MainActor.run { OpenClawChatViewModel(sessionKey: "main", transport: transport) } + + await MainActor.run { vm.load() } + try await waitUntil("bootstrap") { await MainActor.run { vm.healthOK && vm.sessionId == sessionId } } + + await MainActor.run { + vm.input = "hi" + vm.send() + } + try await waitUntil("pending run starts") { await MainActor.run { vm.pendingRunCount == 1 } } + + transport.emit( + .agent( + OpenClawAgentEventPayload( + runId: sessionId, + seq: 1, + stream: "assistant", + ts: Int(Date().timeIntervalSince1970 * 1000), + data: ["text": AnyCodable("streaming…")]))) + + try await waitUntil("assistant stream visible") { + await MainActor.run { vm.streamingAssistantText == "streaming…" } + } + + transport.emit( + .agent( + OpenClawAgentEventPayload( + runId: sessionId, + seq: 2, + stream: "tool", + ts: Int(Date().timeIntervalSince1970 * 1000), + data: [ + "phase": AnyCodable("start"), + "name": AnyCodable("demo"), + "toolCallId": AnyCodable("t1"), + "args": AnyCodable(["x": 1]), + ]))) + + try await waitUntil("tool call pending") { await MainActor.run { vm.pendingToolCalls.count == 1 } } + + let runId = try #require(await transport.lastSentRunId()) + transport.emit( + .chat( + OpenClawChatEventPayload( + runId: runId, + sessionKey: "main", + state: "final", + message: nil, + errorMessage: nil))) + + try await waitUntil("pending run clears") { await MainActor.run { vm.pendingRunCount == 0 } } + try await waitUntil("history refresh") { + await MainActor.run { vm.messages.contains(where: { $0.role == "assistant" }) } + } + #expect(await MainActor.run { vm.streamingAssistantText } == nil) + #expect(await MainActor.run { vm.pendingToolCalls.isEmpty }) + } + + @Test func clearsStreamingOnExternalFinalEvent() async throws { + let sessionId = "sess-main" + let history = OpenClawChatHistoryPayload( + sessionKey: "main", + sessionId: sessionId, + messages: [], + thinkingLevel: "off") + let transport = TestChatTransport(historyResponses: [history, history]) + let vm = await MainActor.run { OpenClawChatViewModel(sessionKey: "main", transport: transport) } + + await MainActor.run { vm.load() } + try await waitUntil("bootstrap") { await MainActor.run { vm.healthOK && vm.sessionId == sessionId } } + + transport.emit( + .agent( + OpenClawAgentEventPayload( + runId: sessionId, + seq: 1, + stream: "assistant", + ts: Int(Date().timeIntervalSince1970 * 1000), + data: ["text": AnyCodable("external stream")]))) + + transport.emit( + .agent( + OpenClawAgentEventPayload( + runId: sessionId, + seq: 2, + stream: "tool", + ts: Int(Date().timeIntervalSince1970 * 1000), + data: [ + "phase": AnyCodable("start"), + "name": AnyCodable("demo"), + "toolCallId": AnyCodable("t1"), + "args": AnyCodable(["x": 1]), + ]))) + + try await waitUntil("streaming active") { + await MainActor.run { vm.streamingAssistantText == "external stream" } + } + try await waitUntil("tool call pending") { await MainActor.run { vm.pendingToolCalls.count == 1 } } + + transport.emit( + .chat( + OpenClawChatEventPayload( + runId: "other-run", + sessionKey: "main", + state: "final", + message: nil, + errorMessage: nil))) + + try await waitUntil("streaming cleared") { await MainActor.run { vm.streamingAssistantText == nil } } + #expect(await MainActor.run { vm.pendingToolCalls.isEmpty }) + } + + @Test func sessionChoicesPreferMainAndRecent() async throws { + let now = Date().timeIntervalSince1970 * 1000 + let recent = now - (2 * 60 * 60 * 1000) + let recentOlder = now - (5 * 60 * 60 * 1000) + let stale = now - (26 * 60 * 60 * 1000) + let history = OpenClawChatHistoryPayload( + sessionKey: "main", + sessionId: "sess-main", + messages: [], + thinkingLevel: "off") + let sessions = OpenClawChatSessionsListResponse( + ts: now, + path: nil, + count: 4, + defaults: nil, + sessions: [ + OpenClawChatSessionEntry( + key: "recent-1", + kind: nil, + displayName: nil, + surface: nil, + subject: nil, + room: nil, + space: nil, + updatedAt: recent, + sessionId: nil, + systemSent: nil, + abortedLastRun: nil, + thinkingLevel: nil, + verboseLevel: nil, + inputTokens: nil, + outputTokens: nil, + totalTokens: nil, + model: nil, + contextTokens: nil), + OpenClawChatSessionEntry( + key: "main", + kind: nil, + displayName: nil, + surface: nil, + subject: nil, + room: nil, + space: nil, + updatedAt: stale, + sessionId: nil, + systemSent: nil, + abortedLastRun: nil, + thinkingLevel: nil, + verboseLevel: nil, + inputTokens: nil, + outputTokens: nil, + totalTokens: nil, + model: nil, + contextTokens: nil), + OpenClawChatSessionEntry( + key: "recent-2", + kind: nil, + displayName: nil, + surface: nil, + subject: nil, + room: nil, + space: nil, + updatedAt: recentOlder, + sessionId: nil, + systemSent: nil, + abortedLastRun: nil, + thinkingLevel: nil, + verboseLevel: nil, + inputTokens: nil, + outputTokens: nil, + totalTokens: nil, + model: nil, + contextTokens: nil), + OpenClawChatSessionEntry( + key: "old-1", + kind: nil, + displayName: nil, + surface: nil, + subject: nil, + room: nil, + space: nil, + updatedAt: stale, + sessionId: nil, + systemSent: nil, + abortedLastRun: nil, + thinkingLevel: nil, + verboseLevel: nil, + inputTokens: nil, + outputTokens: nil, + totalTokens: nil, + model: nil, + contextTokens: nil), + ]) + + let transport = TestChatTransport( + historyResponses: [history], + sessionsResponses: [sessions]) + let vm = await MainActor.run { OpenClawChatViewModel(sessionKey: "main", transport: transport) } + await MainActor.run { vm.load() } + try await waitUntil("sessions loaded") { await MainActor.run { !vm.sessions.isEmpty } } + + let keys = await MainActor.run { vm.sessionChoices.map(\.key) } + #expect(keys == ["main", "recent-1", "recent-2"]) + } + + @Test func sessionChoicesIncludeCurrentWhenMissing() async throws { + let now = Date().timeIntervalSince1970 * 1000 + let recent = now - (30 * 60 * 1000) + let history = OpenClawChatHistoryPayload( + sessionKey: "custom", + sessionId: "sess-custom", + messages: [], + thinkingLevel: "off") + let sessions = OpenClawChatSessionsListResponse( + ts: now, + path: nil, + count: 1, + defaults: nil, + sessions: [ + OpenClawChatSessionEntry( + key: "main", + kind: nil, + displayName: nil, + surface: nil, + subject: nil, + room: nil, + space: nil, + updatedAt: recent, + sessionId: nil, + systemSent: nil, + abortedLastRun: nil, + thinkingLevel: nil, + verboseLevel: nil, + inputTokens: nil, + outputTokens: nil, + totalTokens: nil, + model: nil, + contextTokens: nil), + ]) + + let transport = TestChatTransport( + historyResponses: [history], + sessionsResponses: [sessions]) + let vm = await MainActor.run { OpenClawChatViewModel(sessionKey: "custom", transport: transport) } + await MainActor.run { vm.load() } + try await waitUntil("sessions loaded") { await MainActor.run { !vm.sessions.isEmpty } } + + let keys = await MainActor.run { vm.sessionChoices.map(\.key) } + #expect(keys == ["main", "custom"]) + } + + @Test func clearsStreamingOnExternalErrorEvent() async throws { + let sessionId = "sess-main" + let history = OpenClawChatHistoryPayload( + sessionKey: "main", + sessionId: sessionId, + messages: [], + thinkingLevel: "off") + let transport = TestChatTransport(historyResponses: [history, history]) + let vm = await MainActor.run { OpenClawChatViewModel(sessionKey: "main", transport: transport) } + + await MainActor.run { vm.load() } + try await waitUntil("bootstrap") { await MainActor.run { vm.healthOK && vm.sessionId == sessionId } } + + transport.emit( + .agent( + OpenClawAgentEventPayload( + runId: sessionId, + seq: 1, + stream: "assistant", + ts: Int(Date().timeIntervalSince1970 * 1000), + data: ["text": AnyCodable("external stream")]))) + + try await waitUntil("streaming active") { + await MainActor.run { vm.streamingAssistantText == "external stream" } + } + + transport.emit( + .chat( + OpenClawChatEventPayload( + runId: "other-run", + sessionKey: "main", + state: "error", + message: nil, + errorMessage: "boom"))) + + try await waitUntil("streaming cleared") { await MainActor.run { vm.streamingAssistantText == nil } } + } + + @Test func abortRequestsDoNotClearPendingUntilAbortedEvent() async throws { + let sessionId = "sess-main" + let history = OpenClawChatHistoryPayload( + sessionKey: "main", + sessionId: sessionId, + messages: [], + thinkingLevel: "off") + let transport = TestChatTransport(historyResponses: [history, history]) + let vm = await MainActor.run { OpenClawChatViewModel(sessionKey: "main", transport: transport) } + + await MainActor.run { vm.load() } + try await waitUntil("bootstrap") { await MainActor.run { vm.healthOK && vm.sessionId == sessionId } } + + await MainActor.run { + vm.input = "hi" + vm.send() + } + try await waitUntil("pending run starts") { await MainActor.run { vm.pendingRunCount == 1 } } + + let runId = try #require(await transport.lastSentRunId()) + await MainActor.run { vm.abort() } + + try await waitUntil("abortRun called") { + let ids = await transport.abortedRunIds() + return ids == [runId] + } + + // Pending remains until the gateway broadcasts an aborted/final chat event. + #expect(await MainActor.run { vm.pendingRunCount } == 1) + + transport.emit( + .chat( + OpenClawChatEventPayload( + runId: runId, + sessionKey: "main", + state: "aborted", + message: nil, + errorMessage: nil))) + + try await waitUntil("pending run clears") { await MainActor.run { vm.pendingRunCount == 0 } } + } +} diff --git a/apps/shared/OpenClawKit/Tests/OpenClawKitTests/ElevenLabsTTSValidationTests.swift b/apps/shared/OpenClawKit/Tests/OpenClawKitTests/ElevenLabsTTSValidationTests.swift new file mode 100644 index 0000000000000000000000000000000000000000..1d672db353f1160960def3f32b23537400c6c653 --- /dev/null +++ b/apps/shared/OpenClawKit/Tests/OpenClawKitTests/ElevenLabsTTSValidationTests.swift @@ -0,0 +1,19 @@ +import XCTest +@testable import OpenClawKit + +final class ElevenLabsTTSValidationTests: XCTestCase { + func testValidatedOutputFormatAllowsOnlyMp3Presets() { + XCTAssertEqual(ElevenLabsTTSClient.validatedOutputFormat("mp3_44100_128"), "mp3_44100_128") + XCTAssertEqual(ElevenLabsTTSClient.validatedOutputFormat("pcm_16000"), "pcm_16000") + } + + func testValidatedLanguageAcceptsTwoLetterCodes() { + XCTAssertEqual(ElevenLabsTTSClient.validatedLanguage("EN"), "en") + XCTAssertNil(ElevenLabsTTSClient.validatedLanguage("eng")) + } + + func testValidatedNormalizeAcceptsKnownValues() { + XCTAssertEqual(ElevenLabsTTSClient.validatedNormalize("AUTO"), "auto") + XCTAssertNil(ElevenLabsTTSClient.validatedNormalize("maybe")) + } +} diff --git a/apps/shared/OpenClawKit/Tests/OpenClawKitTests/GatewayNodeSessionTests.swift b/apps/shared/OpenClawKit/Tests/OpenClawKitTests/GatewayNodeSessionTests.swift new file mode 100644 index 0000000000000000000000000000000000000000..91e3096159114ba7484fdc3fc3f8f346f27af4fa --- /dev/null +++ b/apps/shared/OpenClawKit/Tests/OpenClawKitTests/GatewayNodeSessionTests.swift @@ -0,0 +1,56 @@ +import Foundation +import Testing +@testable import OpenClawKit +import OpenClawProtocol + +struct GatewayNodeSessionTests { + @Test + func invokeWithTimeoutReturnsUnderlyingResponseBeforeTimeout() async { + let request = BridgeInvokeRequest(id: "1", command: "x", paramsJSON: nil) + let response = await GatewayNodeSession.invokeWithTimeout( + request: request, + timeoutMs: 50, + onInvoke: { req in + #expect(req.id == "1") + return BridgeInvokeResponse(id: req.id, ok: true, payloadJSON: "{}", error: nil) + } + ) + + #expect(response.ok == true) + #expect(response.error == nil) + #expect(response.payloadJSON == "{}") + } + + @Test + func invokeWithTimeoutReturnsTimeoutError() async { + let request = BridgeInvokeRequest(id: "abc", command: "x", paramsJSON: nil) + let response = await GatewayNodeSession.invokeWithTimeout( + request: request, + timeoutMs: 10, + onInvoke: { _ in + try? await Task.sleep(nanoseconds: 200_000_000) // 200ms + return BridgeInvokeResponse(id: "abc", ok: true, payloadJSON: "{}", error: nil) + } + ) + + #expect(response.ok == false) + #expect(response.error?.code == .unavailable) + #expect(response.error?.message.contains("timed out") == true) + } + + @Test + func invokeWithTimeoutZeroDisablesTimeout() async { + let request = BridgeInvokeRequest(id: "1", command: "x", paramsJSON: nil) + let response = await GatewayNodeSession.invokeWithTimeout( + request: request, + timeoutMs: 0, + onInvoke: { req in + try? await Task.sleep(nanoseconds: 5_000_000) + return BridgeInvokeResponse(id: req.id, ok: true, payloadJSON: nil, error: nil) + } + ) + + #expect(response.ok == true) + #expect(response.error == nil) + } +} diff --git a/apps/shared/OpenClawKit/Tests/OpenClawKitTests/JPEGTranscoderTests.swift b/apps/shared/OpenClawKit/Tests/OpenClawKitTests/JPEGTranscoderTests.swift new file mode 100644 index 0000000000000000000000000000000000000000..5070a8b14e0d88babc405b303cf4ab69d70e27c2 --- /dev/null +++ b/apps/shared/OpenClawKit/Tests/OpenClawKitTests/JPEGTranscoderTests.swift @@ -0,0 +1,129 @@ +import OpenClawKit +import CoreGraphics +import ImageIO +import Testing +import UniformTypeIdentifiers + +@Suite struct JPEGTranscoderTests { + private func makeSolidJPEG(width: Int, height: Int, orientation: Int? = nil) throws -> Data { + let cs = CGColorSpaceCreateDeviceRGB() + let bitmapInfo = CGImageAlphaInfo.premultipliedLast.rawValue + guard + let ctx = CGContext( + data: nil, + width: width, + height: height, + bitsPerComponent: 8, + bytesPerRow: 0, + space: cs, + bitmapInfo: bitmapInfo) + else { + throw NSError(domain: "JPEGTranscoderTests", code: 1) + } + + ctx.setFillColor(red: 1, green: 0, blue: 0, alpha: 1) + ctx.fill(CGRect(x: 0, y: 0, width: width, height: height)) + guard let img = ctx.makeImage() else { + throw NSError(domain: "JPEGTranscoderTests", code: 5) + } + + let out = NSMutableData() + guard let dest = CGImageDestinationCreateWithData(out, UTType.jpeg.identifier as CFString, 1, nil) else { + throw NSError(domain: "JPEGTranscoderTests", code: 2) + } + + var props: [CFString: Any] = [ + kCGImageDestinationLossyCompressionQuality: 1.0, + ] + if let orientation { + props[kCGImagePropertyOrientation] = orientation + } + + CGImageDestinationAddImage(dest, img, props as CFDictionary) + guard CGImageDestinationFinalize(dest) else { + throw NSError(domain: "JPEGTranscoderTests", code: 3) + } + + return out as Data + } + + private func makeNoiseJPEG(width: Int, height: Int) throws -> Data { + let bytesPerPixel = 4 + let byteCount = width * height * bytesPerPixel + var data = Data(count: byteCount) + let cs = CGColorSpaceCreateDeviceRGB() + let bitmapInfo = CGImageAlphaInfo.premultipliedLast.rawValue + + let out = try data.withUnsafeMutableBytes { rawBuffer -> Data in + guard let base = rawBuffer.baseAddress?.assumingMemoryBound(to: UInt8.self) else { + throw NSError(domain: "JPEGTranscoderTests", code: 6) + } + for idx in 0.. 0) + } + + @Test func doesNotUpscaleWhenSmallerThanMaxWidthPx() throws { + let input = try makeSolidJPEG(width: 800, height: 600) + let out = try JPEGTranscoder.transcodeToJPEG(imageData: input, maxWidthPx: 1600, quality: 0.9) + #expect(out.widthPx == 800) + #expect(out.heightPx == 600) + } + + @Test func normalizesOrientationAndUsesOrientedWidthForMaxWidthPx() throws { + // Encode a landscape image but mark it rotated 90° (orientation 6). Oriented width becomes 1000. + let input = try makeSolidJPEG(width: 2000, height: 1000, orientation: 6) + let out = try JPEGTranscoder.transcodeToJPEG(imageData: input, maxWidthPx: 1600, quality: 0.9) + #expect(out.widthPx == 1000) + #expect(out.heightPx == 2000) + } + + @Test func respectsMaxBytes() throws { + let input = try makeNoiseJPEG(width: 1600, height: 1200) + let out = try JPEGTranscoder.transcodeToJPEG( + imageData: input, + maxWidthPx: 1600, + quality: 0.95, + maxBytes: 180_000) + #expect(out.data.count <= 180_000) + } +} diff --git a/apps/shared/OpenClawKit/Tests/OpenClawKitTests/TalkDirectiveTests.swift b/apps/shared/OpenClawKit/Tests/OpenClawKitTests/TalkDirectiveTests.swift new file mode 100644 index 0000000000000000000000000000000000000000..11565ac7448730e86c3a355b592f6ff18e4f831d --- /dev/null +++ b/apps/shared/OpenClawKit/Tests/OpenClawKitTests/TalkDirectiveTests.swift @@ -0,0 +1,74 @@ +import XCTest +@testable import OpenClawKit + +final class TalkDirectiveTests: XCTestCase { + func testParsesDirectiveAndStripsLine() { + let text = """ + {"voice":"abc123","once":true} + Hello there. + """ + let result = TalkDirectiveParser.parse(text) + XCTAssertEqual(result.directive?.voiceId, "abc123") + XCTAssertEqual(result.directive?.once, true) + XCTAssertEqual(result.stripped, "Hello there.") + } + + func testIgnoresNonDirective() { + let text = "Hello world." + let result = TalkDirectiveParser.parse(text) + XCTAssertNil(result.directive) + XCTAssertEqual(result.stripped, text) + } + + func testKeepsDirectiveLineIfNoRecognizedFields() { + let text = """ + {"unknown":"value"} + Hello. + """ + let result = TalkDirectiveParser.parse(text) + XCTAssertNil(result.directive) + XCTAssertEqual(result.stripped, text) + } + + func testParsesExtendedOptions() { + let text = """ + {"voice_id":"v1","model_id":"m1","rate":200,"stability":0.5,"similarity":0.8,"style":0.2,"speaker_boost":true,"seed":1234,"normalize":"auto","lang":"en","output_format":"mp3_44100_128"} + Hello. + """ + let result = TalkDirectiveParser.parse(text) + XCTAssertEqual(result.directive?.voiceId, "v1") + XCTAssertEqual(result.directive?.modelId, "m1") + XCTAssertEqual(result.directive?.rateWPM, 200) + XCTAssertEqual(result.directive?.stability, 0.5) + XCTAssertEqual(result.directive?.similarity, 0.8) + XCTAssertEqual(result.directive?.style, 0.2) + XCTAssertEqual(result.directive?.speakerBoost, true) + XCTAssertEqual(result.directive?.seed, 1234) + XCTAssertEqual(result.directive?.normalize, "auto") + XCTAssertEqual(result.directive?.language, "en") + XCTAssertEqual(result.directive?.outputFormat, "mp3_44100_128") + XCTAssertEqual(result.stripped, "Hello.") + } + + func testSkipsLeadingEmptyLinesWhenParsingDirective() { + let text = """ + + + {"voice":"abc123"} + Hello there. + """ + let result = TalkDirectiveParser.parse(text) + XCTAssertEqual(result.directive?.voiceId, "abc123") + XCTAssertEqual(result.stripped, "Hello there.") + } + + func testTracksUnknownKeys() { + let text = """ + {"voice":"abc","mystery":"value","extra":1} + Hi. + """ + let result = TalkDirectiveParser.parse(text) + XCTAssertEqual(result.directive?.voiceId, "abc") + XCTAssertEqual(result.unknownKeys, ["extra", "mystery"]) + } +} diff --git a/apps/shared/OpenClawKit/Tests/OpenClawKitTests/TalkHistoryTimestampTests.swift b/apps/shared/OpenClawKit/Tests/OpenClawKitTests/TalkHistoryTimestampTests.swift new file mode 100644 index 0000000000000000000000000000000000000000..e66c4e1e9ca697470f40c7cdc0c6d00b6957eff5 --- /dev/null +++ b/apps/shared/OpenClawKit/Tests/OpenClawKitTests/TalkHistoryTimestampTests.swift @@ -0,0 +1,16 @@ +import XCTest +@testable import OpenClawKit + +final class TalkHistoryTimestampTests: XCTestCase { + func testSecondsTimestampsAreAcceptedWithSmallTolerance() { + XCTAssertTrue(TalkHistoryTimestamp.isAfter(999.6, sinceSeconds: 1000)) + XCTAssertFalse(TalkHistoryTimestamp.isAfter(999.4, sinceSeconds: 1000)) + } + + func testMillisecondsTimestampsAreAcceptedWithSmallTolerance() { + let sinceSeconds = 1_700_000_000.0 + let sinceMs = sinceSeconds * 1000 + XCTAssertTrue(TalkHistoryTimestamp.isAfter(sinceMs - 500, sinceSeconds: sinceSeconds)) + XCTAssertFalse(TalkHistoryTimestamp.isAfter(sinceMs - 501, sinceSeconds: sinceSeconds)) + } +} diff --git a/apps/shared/OpenClawKit/Tests/OpenClawKitTests/TalkPromptBuilderTests.swift b/apps/shared/OpenClawKit/Tests/OpenClawKitTests/TalkPromptBuilderTests.swift new file mode 100644 index 0000000000000000000000000000000000000000..1ca18fdf32d9d2e073c6d4d343500668f9972a7e --- /dev/null +++ b/apps/shared/OpenClawKit/Tests/OpenClawKitTests/TalkPromptBuilderTests.swift @@ -0,0 +1,15 @@ +import XCTest +@testable import OpenClawKit + +final class TalkPromptBuilderTests: XCTestCase { + func testBuildIncludesTranscript() { + let prompt = TalkPromptBuilder.build(transcript: "Hello", interruptedAtSeconds: nil) + XCTAssertTrue(prompt.contains("Talk Mode active.")) + XCTAssertTrue(prompt.hasSuffix("\n\nHello")) + } + + func testBuildIncludesInterruptionLineWhenProvided() { + let prompt = TalkPromptBuilder.build(transcript: "Hi", interruptedAtSeconds: 1.234) + XCTAssertTrue(prompt.contains("Assistant speech interrupted at 1.2s.")) + } +} diff --git a/apps/shared/OpenClawKit/Tests/OpenClawKitTests/ToolDisplayRegistryTests.swift b/apps/shared/OpenClawKit/Tests/OpenClawKitTests/ToolDisplayRegistryTests.swift new file mode 100644 index 0000000000000000000000000000000000000000..dbf38138a4bbe227b691271959cd7e51b6ed7920 --- /dev/null +++ b/apps/shared/OpenClawKit/Tests/OpenClawKitTests/ToolDisplayRegistryTests.swift @@ -0,0 +1,16 @@ +import OpenClawKit +import Foundation +import Testing + +@Suite struct ToolDisplayRegistryTests { + @Test func loadsToolDisplayConfigFromBundle() { + let url = OpenClawKitResources.bundle.url(forResource: "tool-display", withExtension: "json") + #expect(url != nil) + } + + @Test func resolvesKnownToolFromConfig() { + let summary = ToolDisplayRegistry.resolve(name: "bash", args: nil) + #expect(summary.emoji == "🛠️") + #expect(summary.title == "Bash") + } +} diff --git a/apps/shared/OpenClawKit/Tools/CanvasA2UI/bootstrap.js b/apps/shared/OpenClawKit/Tools/CanvasA2UI/bootstrap.js new file mode 100644 index 0000000000000000000000000000000000000000..563adcc3b1d4ae05c93b1b158d969805e3b4a886 --- /dev/null +++ b/apps/shared/OpenClawKit/Tools/CanvasA2UI/bootstrap.js @@ -0,0 +1,490 @@ +import { html, css, LitElement, unsafeCSS } from "lit"; +import { repeat } from "lit/directives/repeat.js"; +import { ContextProvider } from "@lit/context"; + +import { v0_8 } from "@a2ui/lit"; +import "@a2ui/lit/ui"; +import { themeContext } from "@openclaw/a2ui-theme-context"; + +const modalStyles = css` + dialog { + position: fixed; + inset: 0; + width: 100%; + height: 100%; + margin: 0; + padding: 24px; + border: none; + background: rgba(5, 8, 16, 0.65); + backdrop-filter: blur(6px); + display: grid; + place-items: center; + } + + dialog::backdrop { + background: rgba(5, 8, 16, 0.65); + backdrop-filter: blur(6px); + } +`; + +const modalElement = customElements.get("a2ui-modal"); +if (modalElement && Array.isArray(modalElement.styles)) { + modalElement.styles = [...modalElement.styles, modalStyles]; +} + +const emptyClasses = () => ({}); +const textHintStyles = () => ({ h1: {}, h2: {}, h3: {}, h4: {}, h5: {}, body: {}, caption: {} }); + +const isAndroid = /Android/i.test(globalThis.navigator?.userAgent ?? ""); +const cardShadow = isAndroid ? "0 2px 10px rgba(0,0,0,.18)" : "0 10px 30px rgba(0,0,0,.35)"; +const buttonShadow = isAndroid ? "0 2px 10px rgba(6, 182, 212, 0.14)" : "0 10px 25px rgba(6, 182, 212, 0.18)"; +const statusShadow = isAndroid ? "0 2px 10px rgba(0, 0, 0, 0.18)" : "0 10px 24px rgba(0, 0, 0, 0.25)"; +const statusBlur = isAndroid ? "10px" : "14px"; + +const openclawTheme = { + components: { + AudioPlayer: emptyClasses(), + Button: emptyClasses(), + Card: emptyClasses(), + Column: emptyClasses(), + CheckBox: { container: emptyClasses(), element: emptyClasses(), label: emptyClasses() }, + DateTimeInput: { container: emptyClasses(), element: emptyClasses(), label: emptyClasses() }, + Divider: emptyClasses(), + Image: { + all: emptyClasses(), + icon: emptyClasses(), + avatar: emptyClasses(), + smallFeature: emptyClasses(), + mediumFeature: emptyClasses(), + largeFeature: emptyClasses(), + header: emptyClasses(), + }, + Icon: emptyClasses(), + List: emptyClasses(), + Modal: { backdrop: emptyClasses(), element: emptyClasses() }, + MultipleChoice: { container: emptyClasses(), element: emptyClasses(), label: emptyClasses() }, + Row: emptyClasses(), + Slider: { container: emptyClasses(), element: emptyClasses(), label: emptyClasses() }, + Tabs: { container: emptyClasses(), element: emptyClasses(), controls: { all: emptyClasses(), selected: emptyClasses() } }, + Text: { + all: emptyClasses(), + h1: emptyClasses(), + h2: emptyClasses(), + h3: emptyClasses(), + h4: emptyClasses(), + h5: emptyClasses(), + caption: emptyClasses(), + body: emptyClasses(), + }, + TextField: { container: emptyClasses(), element: emptyClasses(), label: emptyClasses() }, + Video: emptyClasses(), + }, + elements: { + a: emptyClasses(), + audio: emptyClasses(), + body: emptyClasses(), + button: emptyClasses(), + h1: emptyClasses(), + h2: emptyClasses(), + h3: emptyClasses(), + h4: emptyClasses(), + h5: emptyClasses(), + iframe: emptyClasses(), + input: emptyClasses(), + p: emptyClasses(), + pre: emptyClasses(), + textarea: emptyClasses(), + video: emptyClasses(), + }, + markdown: { + p: [], + h1: [], + h2: [], + h3: [], + h4: [], + h5: [], + ul: [], + ol: [], + li: [], + a: [], + strong: [], + em: [], + }, + additionalStyles: { + Card: { + background: "linear-gradient(180deg, rgba(255,255,255,.06), rgba(255,255,255,.03))", + border: "1px solid rgba(255,255,255,.09)", + borderRadius: "14px", + padding: "14px", + boxShadow: cardShadow, + }, + Modal: { + background: "rgba(12, 16, 24, 0.92)", + border: "1px solid rgba(255,255,255,.12)", + borderRadius: "16px", + padding: "16px", + boxShadow: "0 30px 80px rgba(0,0,0,.6)", + width: "min(520px, calc(100vw - 48px))", + }, + Column: { gap: "10px" }, + Row: { gap: "10px", alignItems: "center" }, + Divider: { opacity: "0.25" }, + Button: { + background: "linear-gradient(135deg, #22c55e 0%, #06b6d4 100%)", + border: "0", + borderRadius: "12px", + padding: "10px 14px", + color: "#071016", + fontWeight: "650", + cursor: "pointer", + boxShadow: buttonShadow, + }, + Text: { + ...textHintStyles(), + h1: { fontSize: "20px", fontWeight: "750", margin: "0 0 6px 0" }, + h2: { fontSize: "16px", fontWeight: "700", margin: "0 0 6px 0" }, + body: { fontSize: "13px", lineHeight: "1.4" }, + caption: { opacity: "0.8" }, + }, + TextField: { display: "grid", gap: "6px" }, + Image: { borderRadius: "12px" }, + }, +}; + +class OpenClawA2UIHost extends LitElement { + static properties = { + surfaces: { state: true }, + pendingAction: { state: true }, + toast: { state: true }, + }; + + #processor = v0_8.Data.createSignalA2uiMessageProcessor(); + themeProvider = new ContextProvider(this, { + context: themeContext, + initialValue: openclawTheme, + }); + + surfaces = []; + pendingAction = null; + toast = null; + #statusListener = null; + + static styles = css` + :host { + display: block; + height: 100%; + position: relative; + box-sizing: border-box; + padding: + var(--openclaw-a2ui-inset-top, 0px) + var(--openclaw-a2ui-inset-right, 0px) + var(--openclaw-a2ui-inset-bottom, 0px) + var(--openclaw-a2ui-inset-left, 0px); + } + + #surfaces { + display: grid; + grid-template-columns: 1fr; + gap: 12px; + height: 100%; + overflow: auto; + padding-bottom: var(--openclaw-a2ui-scroll-pad-bottom, 0px); + } + + .status { + position: absolute; + left: 50%; + transform: translateX(-50%); + top: var(--openclaw-a2ui-status-top, 12px); + display: inline-flex; + align-items: center; + gap: 8px; + padding: 8px 10px; + border-radius: 12px; + background: rgba(0, 0, 0, 0.45); + border: 1px solid rgba(255, 255, 255, 0.18); + color: rgba(255, 255, 255, 0.92); + font: 13px/1.2 system-ui, -apple-system, BlinkMacSystemFont, "Roboto", sans-serif; + pointer-events: none; + backdrop-filter: blur(${unsafeCSS(statusBlur)}); + -webkit-backdrop-filter: blur(${unsafeCSS(statusBlur)}); + box-shadow: ${unsafeCSS(statusShadow)}; + z-index: 5; + } + + .toast { + position: absolute; + left: 50%; + transform: translateX(-50%); + bottom: var(--openclaw-a2ui-toast-bottom, 12px); + display: inline-flex; + align-items: center; + gap: 8px; + padding: 8px 10px; + border-radius: 12px; + background: rgba(0, 0, 0, 0.45); + border: 1px solid rgba(255, 255, 255, 0.18); + color: rgba(255, 255, 255, 0.92); + font: 13px/1.2 system-ui, -apple-system, BlinkMacSystemFont, "Roboto", sans-serif; + pointer-events: none; + backdrop-filter: blur(${unsafeCSS(statusBlur)}); + -webkit-backdrop-filter: blur(${unsafeCSS(statusBlur)}); + box-shadow: ${unsafeCSS(statusShadow)}; + z-index: 5; + } + + .toast.error { + border-color: rgba(255, 109, 109, 0.35); + color: rgba(255, 223, 223, 0.98); + } + + .empty { + position: absolute; + left: 50%; + transform: translateX(-50%); + top: var(--openclaw-a2ui-empty-top, var(--openclaw-a2ui-status-top, 12px)); + text-align: center; + opacity: 0.8; + padding: 10px 12px; + pointer-events: none; + } + + .empty-title { + font-weight: 700; + margin-bottom: 6px; + } + + .spinner { + width: 12px; + height: 12px; + border-radius: 999px; + border: 2px solid rgba(255, 255, 255, 0.25); + border-top-color: rgba(255, 255, 255, 0.92); + animation: spin 0.75s linear infinite; + } + + @keyframes spin { + from { + transform: rotate(0deg); + } + to { + transform: rotate(360deg); + } + } + `; + + connectedCallback() { + super.connectedCallback(); + const api = { + applyMessages: (messages) => this.applyMessages(messages), + reset: () => this.reset(), + getSurfaces: () => Array.from(this.#processor.getSurfaces().keys()), + }; + globalThis.openclawA2UI = api; + this.addEventListener("a2uiaction", (evt) => this.#handleA2UIAction(evt)); + this.#statusListener = (evt) => this.#handleActionStatus(evt); + for (const eventName of ["openclaw:a2ui-action-status"]) { + globalThis.addEventListener(eventName, this.#statusListener); + } + this.#syncSurfaces(); + } + + disconnectedCallback() { + super.disconnectedCallback(); + if (this.#statusListener) { + for (const eventName of ["openclaw:a2ui-action-status"]) { + globalThis.removeEventListener(eventName, this.#statusListener); + } + this.#statusListener = null; + } + } + + #makeActionId() { + return globalThis.crypto?.randomUUID?.() ?? `a2ui_${Date.now()}_${Math.random().toString(16).slice(2)}`; + } + + #setToast(text, kind = "ok", timeoutMs = 1400) { + const toast = { text, kind, expiresAt: Date.now() + timeoutMs }; + this.toast = toast; + this.requestUpdate(); + setTimeout(() => { + if (this.toast === toast) { + this.toast = null; + this.requestUpdate(); + } + }, timeoutMs + 30); + } + + #handleActionStatus(evt) { + const detail = evt?.detail ?? null; + if (!detail || typeof detail.id !== "string") {return;} + if (!this.pendingAction || this.pendingAction.id !== detail.id) {return;} + + if (detail.ok) { + this.pendingAction = { ...this.pendingAction, phase: "sent", sentAt: Date.now() }; + } else { + const msg = typeof detail.error === "string" && detail.error ? detail.error : "send failed"; + this.pendingAction = { ...this.pendingAction, phase: "error", error: msg }; + this.#setToast(`Failed: ${msg}`, "error", 4500); + } + this.requestUpdate(); + } + + #handleA2UIAction(evt) { + const payload = evt?.detail ?? evt?.payload ?? null; + if (!payload || payload.eventType !== "a2ui.action") { + return; + } + + const action = payload.action; + const name = action?.name; + if (!name) { + return; + } + + const sourceComponentId = payload.sourceComponentId ?? ""; + const surfaces = this.#processor.getSurfaces(); + + let surfaceId = null; + let sourceNode = null; + for (const [sid, surface] of surfaces.entries()) { + const node = surface?.components?.get?.(sourceComponentId) ?? null; + if (node) { + surfaceId = sid; + sourceNode = node; + break; + } + } + + const context = {}; + const ctxItems = Array.isArray(action?.context) ? action.context : []; + for (const item of ctxItems) { + const key = item?.key; + const value = item?.value ?? null; + if (!key || !value) {continue;} + + if (typeof value.path === "string") { + const resolved = sourceNode + ? this.#processor.getData(sourceNode, value.path, surfaceId ?? undefined) + : null; + context[key] = resolved; + continue; + } + if (Object.prototype.hasOwnProperty.call(value, "literalString")) { + context[key] = value.literalString ?? ""; + continue; + } + if (Object.prototype.hasOwnProperty.call(value, "literalNumber")) { + context[key] = value.literalNumber ?? 0; + continue; + } + if (Object.prototype.hasOwnProperty.call(value, "literalBoolean")) { + context[key] = value.literalBoolean ?? false; + continue; + } + } + + const actionId = this.#makeActionId(); + this.pendingAction = { id: actionId, name, phase: "sending", startedAt: Date.now() }; + this.requestUpdate(); + + const userAction = { + id: actionId, + name, + surfaceId: surfaceId ?? "main", + sourceComponentId, + timestamp: new Date().toISOString(), + ...(Object.keys(context).length ? { context } : {}), + }; + + globalThis.__openclawLastA2UIAction = userAction; + + const handler = + globalThis.webkit?.messageHandlers?.openclawCanvasA2UIAction ?? + globalThis.openclawCanvasA2UIAction; + if (handler?.postMessage) { + try { + // WebKit message handlers support structured objects; Android's JS interface expects strings. + if (handler === globalThis.openclawCanvasA2UIAction) { + handler.postMessage(JSON.stringify({ userAction })); + } else { + handler.postMessage({ userAction }); + } + } catch (e) { + const msg = String(e?.message ?? e); + this.pendingAction = { id: actionId, name, phase: "error", startedAt: Date.now(), error: msg }; + this.#setToast(`Failed: ${msg}`, "error", 4500); + } + } else { + this.pendingAction = { id: actionId, name, phase: "error", startedAt: Date.now(), error: "missing native bridge" }; + this.#setToast("Failed: missing native bridge", "error", 4500); + } + } + + applyMessages(messages) { + if (!Array.isArray(messages)) { + throw new Error("A2UI: expected messages array"); + } + this.#processor.processMessages(messages); + this.#syncSurfaces(); + if (this.pendingAction?.phase === "sent") { + this.#setToast(`Updated: ${this.pendingAction.name}`, "ok", 1100); + this.pendingAction = null; + } + this.requestUpdate(); + return { ok: true, surfaces: this.surfaces.map(([id]) => id) }; + } + + reset() { + this.#processor.clearSurfaces(); + this.#syncSurfaces(); + this.pendingAction = null; + this.requestUpdate(); + return { ok: true }; + } + + #syncSurfaces() { + this.surfaces = Array.from(this.#processor.getSurfaces().entries()); + } + + render() { + if (this.surfaces.length === 0) { + return html`
+
Canvas (A2UI)
+
Waiting for A2UI messages…
+
`; + } + + const statusText = + this.pendingAction?.phase === "sent" + ? `Working: ${this.pendingAction.name}` + : this.pendingAction?.phase === "sending" + ? `Sending: ${this.pendingAction.name}` + : this.pendingAction?.phase === "error" + ? `Failed: ${this.pendingAction.name}` + : ""; + + return html` + ${this.pendingAction && this.pendingAction.phase !== "error" + ? html`
${statusText}
` + : ""} + ${this.toast + ? html`
${this.toast.text}
` + : ""} +
+ ${repeat( + this.surfaces, + ([surfaceId]) => surfaceId, + ([surfaceId, surface]) => html`` + )} +
`; + } +} + +if (!customElements.get("openclaw-a2ui-host")) { + customElements.define("openclaw-a2ui-host", OpenClawA2UIHost); +} diff --git a/apps/shared/OpenClawKit/Tools/CanvasA2UI/rolldown.config.mjs b/apps/shared/OpenClawKit/Tools/CanvasA2UI/rolldown.config.mjs new file mode 100644 index 0000000000000000000000000000000000000000..dbd4b86fff68d4a8b151095aa04c6fa3e98e6e94 --- /dev/null +++ b/apps/shared/OpenClawKit/Tools/CanvasA2UI/rolldown.config.mjs @@ -0,0 +1,45 @@ +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { defineConfig } from "rolldown"; + +const here = path.dirname(fileURLToPath(import.meta.url)); +const repoRoot = path.resolve(here, "../../../../.."); +const fromHere = (p) => path.resolve(here, p); +const outputFile = path.resolve( + here, + "../../../../..", + "src", + "canvas-host", + "a2ui", + "a2ui.bundle.js", +); + +const a2uiLitDist = path.resolve(repoRoot, "vendor/a2ui/renderers/lit/dist/src"); +const a2uiThemeContext = path.resolve(a2uiLitDist, "0.8/ui/context/theme.js"); + +export default defineConfig({ + input: fromHere("bootstrap.js"), + experimental: { + attachDebugInfo: "none", + }, + treeshake: false, + resolve: { + alias: { + "@a2ui/lit": path.resolve(a2uiLitDist, "index.js"), + "@a2ui/lit/ui": path.resolve(a2uiLitDist, "0.8/ui/ui.js"), + "@openclaw/a2ui-theme-context": a2uiThemeContext, + "@lit/context": path.resolve(repoRoot, "node_modules/@lit/context/index.js"), + "@lit/context/": path.resolve(repoRoot, "node_modules/@lit/context/"), + "@lit-labs/signals": path.resolve(repoRoot, "node_modules/@lit-labs/signals/index.js"), + "@lit-labs/signals/": path.resolve(repoRoot, "node_modules/@lit-labs/signals/"), + lit: path.resolve(repoRoot, "node_modules/lit/index.js"), + "lit/": path.resolve(repoRoot, "node_modules/lit/"), + }, + }, + output: { + file: outputFile, + format: "esm", + codeSplitting: false, + sourcemap: false, + }, +});