feat: cross-device support — all Apple devices, cloud tool delegation, PWA
Browse files- Fix uninstall permission errors (sudo for /Applications)
- Fix mac app not opening (hardcode JARVIS_DIR in C launcher)
- Fix mic staying on after 'bye' (standby mode, menu bar indicator)
- Fix calendar permission spam (cached permission checks)
- Fix app opening (alias mapping: camera→FaceTime, vscode→VS Code, etc.)
- Add startup automation permission requests from JARVIS process
- Server error retry prompt in conversation loop
Cloud tool delegation:
- Tag 78 macOS-only tools (osascript/subprocess)
- Auto-delegate to connected Mac via WebSocket when server runs on Linux
- Listener connects to server WebSocket, executes tools locally, returns results
- 26 tools run directly in cloud (web search, weather, tasks, memory, etc.)
iPhone/iPad/Apple Watch:
- Siri Shortcut now hits HF Space (not localhost) — works on all Apple devices
- Shortcut: dictate → POST /api/ask → speak response → notification
- PWA: apple-touch-icons (152/167/180px), safe area insets, iPad layout
- iOS install prompt with device detection, don't-show-again option
- manifest.json with full icon set, service worker v2
WebView app:
- Try local server → HF Space → Render → error page
- No local HTML dependency, loads server UI directly
- .gitignore +1 -0
- install.sh +46 -46
- install_siri_shortcut.py +206 -79
- jarvis_app.py +51 -6
- jarvis_listener.py +231 -7
- server.py +4 -1
- static/apple-touch-icon-152.png +0 -0
- static/apple-touch-icon-167.png +0 -0
- static/apple-touch-icon.png +0 -0
- static/index.html +51 -15
- static/manifest.json +20 -1
- static/sw.js +10 -2
- tests/test_app_automation.py +50 -2
- tools/__init__.py +112 -2
- tools/app_automation.py +30 -0
- tools/builtin.py +37 -6
- uninstall.sh +17 -12
|
@@ -13,3 +13,4 @@ JARVIS-Listener.app/
|
|
| 13 |
JARVIS.app/
|
| 14 |
*.icns
|
| 15 |
*.shortcut
|
|
|
|
|
|
| 13 |
JARVIS.app/
|
| 14 |
*.icns
|
| 15 |
*.shortcut
|
| 16 |
+
.device_id
|
|
@@ -90,28 +90,26 @@ fi
|
|
| 90 |
|
| 91 |
# Create the native launcher binary
|
| 92 |
# A compiled binary means macOS TCC properly identifies this app for mic permission
|
| 93 |
-
|
|
|
|
| 94 |
#include <unistd.h>
|
| 95 |
-
#include <libgen.h>
|
| 96 |
-
#include <string.h>
|
| 97 |
#include <stdio.h>
|
| 98 |
#include <stdlib.h>
|
| 99 |
-
|
|
|
|
| 100 |
|
| 101 |
int main(int argc, char *argv[]) {
|
| 102 |
-
char exe_path[4096];
|
| 103 |
-
uint32_t size = sizeof(exe_path);
|
| 104 |
-
_NSGetExecutablePath(exe_path, &size);
|
| 105 |
-
char *dir = dirname(exe_path);
|
| 106 |
-
dir = dirname(dir);
|
| 107 |
-
dir = dirname(dir);
|
| 108 |
-
dir = dirname(dir);
|
| 109 |
char python_path[4096];
|
| 110 |
char script_path[4096];
|
| 111 |
-
snprintf(python_path, sizeof(python_path), "%s/venv/bin/python",
|
| 112 |
-
snprintf(script_path, sizeof(script_path), "%s/jarvis_listener.py",
|
| 113 |
-
chdir(
|
| 114 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 115 |
perror("Failed to launch JARVIS listener");
|
| 116 |
return 1;
|
| 117 |
}
|
|
@@ -196,26 +194,20 @@ if [ -f "$JARVIS_DIR/static/JARVIS.icns" ]; then
|
|
| 196 |
fi
|
| 197 |
|
| 198 |
# Create launcher that runs jarvis_app.py (the WebView)
|
| 199 |
-
|
|
|
|
| 200 |
#include <unistd.h>
|
| 201 |
#include <stdio.h>
|
| 202 |
#include <stdlib.h>
|
| 203 |
-
|
| 204 |
-
#
|
| 205 |
|
| 206 |
int main(int argc, char *argv[]) {
|
| 207 |
-
char exe_path[4096];
|
| 208 |
-
uint32_t size = sizeof(exe_path);
|
| 209 |
-
_NSGetExecutablePath(exe_path, &size);
|
| 210 |
-
char *dir = dirname(exe_path);
|
| 211 |
-
dir = dirname(dir);
|
| 212 |
-
dir = dirname(dir);
|
| 213 |
-
dir = dirname(dir);
|
| 214 |
char python_path[4096];
|
| 215 |
char script_path[4096];
|
| 216 |
-
snprintf(python_path, sizeof(python_path), "%s/venv/bin/python",
|
| 217 |
-
snprintf(script_path, sizeof(script_path), "%s/jarvis_app.py",
|
| 218 |
-
chdir(
|
| 219 |
execl(python_path, python_path, script_path, NULL);
|
| 220 |
perror("Failed to launch JARVIS app");
|
| 221 |
return 1;
|
|
@@ -341,10 +333,18 @@ except Exception as e:
|
|
| 341 |
|
| 342 |
# 6e. AppleScript / Automation (Notes, Calendar, Mail, Finder, etc.)
|
| 343 |
echo " Requesting automation access (Notes, Calendar, Mail, Finder)..."
|
| 344 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 345 |
osascript -e "tell application \"$APP_NAME\" to name" 2>/dev/null && \
|
| 346 |
-
echo " ✓ Automation: $APP_NAME" || \
|
| 347 |
-
echo " [
|
| 348 |
done
|
| 349 |
|
| 350 |
# 6f. Contacts
|
|
@@ -520,28 +520,28 @@ echo " ✓ All services launched (listener includes menu bar)"
|
|
| 520 |
|
| 521 |
echo ""
|
| 522 |
echo " ╔═══════════════════════════════════════╗"
|
| 523 |
-
echo " ║ J.A.R.V.I.S. INSTALLED
|
| 524 |
echo " ║ ║"
|
| 525 |
-
echo " ║ ✓ Server: auto-starts on boot
|
| 526 |
-
echo " ║ ✓ Listener: wake+clap+hotkey+menu
|
| 527 |
-
echo " ║ ✓ JARVIS.app: in /Applications
|
| 528 |
-
echo " ║ ✓ Menu bar: J icon (unified)
|
| 529 |
-
echo " ║ ✓ Siri Shortcut: Hey Siri, JARVIS
|
| 530 |
-
echo " ║ ✓ Clap/Snap: double-clap to wake
|
| 531 |
-
echo " ║ ✓ All permissions: requested
|
| 532 |
echo " ║ ║"
|
| 533 |
echo " ║ Activation: ║"
|
| 534 |
-
echo " ║ • Open JARVIS.app (WebView UI)
|
| 535 |
-
echo " ║ • Say 'Hey Siri, JARVIS'
|
| 536 |
-
echo " ║ • Say 'Jarvis' (local wake word)
|
| 537 |
-
echo " ║ • Ctrl+Shift+J (global hotkey)
|
| 538 |
-
echo " ║ • Double clap or snap
|
| 539 |
-
echo " ║ • Click J in menu bar
|
| 540 |
echo " ║ ║"
|
| 541 |
echo " ║ Conversation mode: ║"
|
| 542 |
echo " ║ • After wake, mic stays on ║"
|
| 543 |
echo " ║ • Ask follow-up questions ║"
|
| 544 |
-
echo " ║ • Say 'goodbye' to end session
|
| 545 |
echo " ║ ║"
|
| 546 |
echo " ║ To uninstall: ║"
|
| 547 |
echo " ║ bash ~/jarvis/uninstall.sh ║"
|
|
|
|
| 90 |
|
| 91 |
# Create the native launcher binary
|
| 92 |
# A compiled binary means macOS TCC properly identifies this app for mic permission
|
| 93 |
+
# Uses JARVIS_DIR hardcoded at build time so it works even if .app is moved
|
| 94 |
+
cat > /tmp/jarvis_launcher.c << CSOURCE
|
| 95 |
#include <unistd.h>
|
|
|
|
|
|
|
| 96 |
#include <stdio.h>
|
| 97 |
#include <stdlib.h>
|
| 98 |
+
|
| 99 |
+
#define JARVIS_DIR "$JARVIS_DIR"
|
| 100 |
|
| 101 |
int main(int argc, char *argv[]) {
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 102 |
char python_path[4096];
|
| 103 |
char script_path[4096];
|
| 104 |
+
snprintf(python_path, sizeof(python_path), "%s/venv/bin/python", JARVIS_DIR);
|
| 105 |
+
snprintf(script_path, sizeof(script_path), "%s/jarvis_listener.py", JARVIS_DIR);
|
| 106 |
+
chdir(JARVIS_DIR);
|
| 107 |
+
/* Forward any extra args (e.g. --no-greet) */
|
| 108 |
+
if (argc > 1) {
|
| 109 |
+
execl(python_path, python_path, script_path, argv[1], NULL);
|
| 110 |
+
} else {
|
| 111 |
+
execl(python_path, python_path, script_path, NULL);
|
| 112 |
+
}
|
| 113 |
perror("Failed to launch JARVIS listener");
|
| 114 |
return 1;
|
| 115 |
}
|
|
|
|
| 194 |
fi
|
| 195 |
|
| 196 |
# Create launcher that runs jarvis_app.py (the WebView)
|
| 197 |
+
# Uses JARVIS_DIR hardcoded at build time so it works from /Applications too
|
| 198 |
+
cat > /tmp/jarvis_app_launcher.c << CSOURCE
|
| 199 |
#include <unistd.h>
|
| 200 |
#include <stdio.h>
|
| 201 |
#include <stdlib.h>
|
| 202 |
+
|
| 203 |
+
#define JARVIS_DIR "$JARVIS_DIR"
|
| 204 |
|
| 205 |
int main(int argc, char *argv[]) {
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 206 |
char python_path[4096];
|
| 207 |
char script_path[4096];
|
| 208 |
+
snprintf(python_path, sizeof(python_path), "%s/venv/bin/python", JARVIS_DIR);
|
| 209 |
+
snprintf(script_path, sizeof(script_path), "%s/jarvis_app.py", JARVIS_DIR);
|
| 210 |
+
chdir(JARVIS_DIR);
|
| 211 |
execl(python_path, python_path, script_path, NULL);
|
| 212 |
perror("Failed to launch JARVIS app");
|
| 213 |
return 1;
|
|
|
|
| 333 |
|
| 334 |
# 6e. AppleScript / Automation (Notes, Calendar, Mail, Finder, etc.)
|
| 335 |
echo " Requesting automation access (Notes, Calendar, Mail, Finder)..."
|
| 336 |
+
echo " NOTE: Automation permissions are requested when JARVIS first starts."
|
| 337 |
+
echo " When JARVIS launches, macOS will prompt you to allow it to control"
|
| 338 |
+
echo " each app. Click 'OK' or 'Allow' for each prompt."
|
| 339 |
+
echo ""
|
| 340 |
+
echo " If you don't see prompts, go to:"
|
| 341 |
+
echo " System Settings → Privacy & Security → Automation → enable JARVIS"
|
| 342 |
+
echo ""
|
| 343 |
+
# Still trigger from Terminal for any that might work
|
| 344 |
+
for APP_NAME in "System Events" "Finder"; do
|
| 345 |
osascript -e "tell application \"$APP_NAME\" to name" 2>/dev/null && \
|
| 346 |
+
echo " ✓ Automation: $APP_NAME (Terminal)" || \
|
| 347 |
+
echo " [i] Automation: $APP_NAME — will prompt when JARVIS starts"
|
| 348 |
done
|
| 349 |
|
| 350 |
# 6f. Contacts
|
|
|
|
| 520 |
|
| 521 |
echo ""
|
| 522 |
echo " ╔═══════════════════════════════════════╗"
|
| 523 |
+
echo " ║ J.A.R.V.I.S. INSTALLED ║"
|
| 524 |
echo " ║ ║"
|
| 525 |
+
echo " ║ ✓ Server: auto-starts on boot ║"
|
| 526 |
+
echo " ║ ✓ Listener: wake+clap+hotkey+menu ║"
|
| 527 |
+
echo " ║ ✓ JARVIS.app: in /Applications ║"
|
| 528 |
+
echo " ║ ✓ Menu bar: J icon (unified) ║"
|
| 529 |
+
echo " ║ ✓ Siri Shortcut: Hey Siri, JARVIS ║"
|
| 530 |
+
echo " ║ ✓ Clap/Snap: double-clap to wake ║"
|
| 531 |
+
echo " ║ ✓ All permissions: requested ║"
|
| 532 |
echo " ║ ║"
|
| 533 |
echo " ║ Activation: ║"
|
| 534 |
+
echo " ║ • Open JARVIS.app (WebView UI) ║"
|
| 535 |
+
echo " ║ • Say 'Hey Siri, JARVIS' ║"
|
| 536 |
+
echo " ║ • Say 'Jarvis' (local wake word) ║"
|
| 537 |
+
echo " ║ • Ctrl+Shift+J (global hotkey) ║"
|
| 538 |
+
echo " ║ • Double clap or snap ║"
|
| 539 |
+
echo " ║ • Click J in menu bar ║"
|
| 540 |
echo " ║ ║"
|
| 541 |
echo " ║ Conversation mode: ║"
|
| 542 |
echo " ║ • After wake, mic stays on ║"
|
| 543 |
echo " ║ • Ask follow-up questions ║"
|
| 544 |
+
echo " ║ • Say 'goodbye' to end session ║"
|
| 545 |
echo " ║ ║"
|
| 546 |
echo " ║ To uninstall: ║"
|
| 547 |
echo " ║ bash ~/jarvis/uninstall.sh ║"
|
|
@@ -1,98 +1,225 @@
|
|
| 1 |
#!/usr/bin/env python3
|
| 2 |
-
"""Generate and install
|
| 3 |
|
| 4 |
-
|
| 5 |
-
|
|
|
|
|
|
|
| 6 |
|
| 7 |
-
|
| 8 |
-
|
|
|
|
|
|
|
|
|
|
| 9 |
"""
|
| 10 |
|
| 11 |
import plistlib
|
| 12 |
import subprocess
|
| 13 |
import os
|
|
|
|
|
|
|
|
|
|
| 14 |
|
| 15 |
JARVIS_DIR = os.path.dirname(os.path.abspath(__file__))
|
| 16 |
-
|
| 17 |
-
|
| 18 |
-
|
| 19 |
-
|
| 20 |
-
|
| 21 |
-
|
| 22 |
-
|
| 23 |
-
|
| 24 |
-
|
| 25 |
-
|
| 26 |
-
|
| 27 |
-
|
| 28 |
-
|
| 29 |
-
|
| 30 |
-
|
| 31 |
-
|
| 32 |
-
"WFWorkflowTypes": [
|
| 33 |
-
"NCWidget", # Notification Center
|
| 34 |
-
"WatchKit", # Apple Watch
|
| 35 |
-
],
|
| 36 |
-
"WFWorkflowInputContentItemClasses": [
|
| 37 |
-
"WFStringContentItem",
|
| 38 |
-
],
|
| 39 |
-
"WFWorkflowActions": [
|
| 40 |
-
# ── Action 1: Set the trigger URL ──
|
| 41 |
{
|
| 42 |
-
"WFWorkflowActionIdentifier": "is.workflow.actions.
|
| 43 |
"WFWorkflowActionParameters": {
|
| 44 |
-
"
|
|
|
|
| 45 |
},
|
| 46 |
},
|
| 47 |
-
# ── Action 2:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 48 |
{
|
| 49 |
"WFWorkflowActionIdentifier": "is.workflow.actions.downloadurl",
|
| 50 |
"WFWorkflowActionParameters": {
|
| 51 |
-
"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 52 |
},
|
| 53 |
},
|
| 54 |
-
|
| 55 |
-
|
| 56 |
-
|
| 57 |
-
|
| 58 |
-
|
| 59 |
-
|
| 60 |
-
|
| 61 |
-
|
| 62 |
-
|
| 63 |
-
|
| 64 |
-
|
| 65 |
-
|
| 66 |
-
|
| 67 |
-
|
| 68 |
-
|
| 69 |
-
|
| 70 |
-
|
| 71 |
-
|
| 72 |
-
|
| 73 |
-
|
| 74 |
-
|
| 75 |
-
|
| 76 |
-
|
| 77 |
-
|
| 78 |
-
|
| 79 |
-
|
| 80 |
-
|
| 81 |
-
|
| 82 |
-
|
| 83 |
-
|
| 84 |
-
|
| 85 |
-
|
| 86 |
-
|
| 87 |
-
|
| 88 |
-
|
| 89 |
-
|
| 90 |
-
|
| 91 |
-
|
| 92 |
-
|
| 93 |
-
|
| 94 |
-
|
| 95 |
-
|
| 96 |
-
|
| 97 |
-
|
| 98 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
#!/usr/bin/env python3
|
| 2 |
+
"""Generate and install JARVIS Siri Shortcuts for ALL Apple devices.
|
| 3 |
|
| 4 |
+
Creates two shortcuts:
|
| 5 |
+
1. "JARVIS" — Full voice assistant. Works on iPhone, iPad, Mac, Apple Watch.
|
| 6 |
+
Say "Hey Siri, JARVIS" → asks what you need → sends to HF Space → speaks result.
|
| 7 |
+
2. "Ask JARVIS" — Quick text/voice input variant for Apple Watch.
|
| 8 |
|
| 9 |
+
Flow: Siri → Shortcut → Dictate/text input → POST to JARVIS server → Speak result.
|
| 10 |
+
|
| 11 |
+
Works on ALL Apple devices because it hits the cloud server (HF Space / Render),
|
| 12 |
+
not localhost. The cloud server delegates macOS-specific tool calls to the
|
| 13 |
+
connected Mac listener via WebSocket.
|
| 14 |
"""
|
| 15 |
|
| 16 |
import plistlib
|
| 17 |
import subprocess
|
| 18 |
import os
|
| 19 |
+
from dotenv import load_dotenv
|
| 20 |
+
|
| 21 |
+
load_dotenv()
|
| 22 |
|
| 23 |
JARVIS_DIR = os.path.dirname(os.path.abspath(__file__))
|
| 24 |
+
HF_SPACE_URL = os.getenv("HF_SPACE_URL", "https://v1deh-jarvis.hf.space")
|
| 25 |
+
RENDER_URL = os.getenv("RENDER_URL", "")
|
| 26 |
+
LOCAL_TRIGGER_URL = "http://127.0.0.1:8111/trigger"
|
| 27 |
+
|
| 28 |
+
# Use cloud URL for cross-device, fall back to local
|
| 29 |
+
JARVIS_API_URL = RENDER_URL or HF_SPACE_URL
|
| 30 |
+
ASK_ENDPOINT = f"{JARVIS_API_URL}/api/ask"
|
| 31 |
+
|
| 32 |
+
# Auth token for the server (if any)
|
| 33 |
+
AUTH_TOKEN = os.getenv("JARVIS_AUTH_TOKEN", "")
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
def build_jarvis_shortcut() -> dict:
|
| 37 |
+
"""Build the main JARVIS shortcut — voice in, voice out, all Apple devices."""
|
| 38 |
+
actions = [
|
| 39 |
+
# ── Action 1: Ask for voice input ("What do you need, sir?") ──
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 40 |
{
|
| 41 |
+
"WFWorkflowActionIdentifier": "is.workflow.actions.ask",
|
| 42 |
"WFWorkflowActionParameters": {
|
| 43 |
+
"WFAskActionPrompt": "What do you need, sir?",
|
| 44 |
+
"WFInputType": "Text",
|
| 45 |
},
|
| 46 |
},
|
| 47 |
+
# ── Action 2: Store the user's input ──
|
| 48 |
+
{
|
| 49 |
+
"WFWorkflowActionIdentifier": "is.workflow.actions.setvariable",
|
| 50 |
+
"WFWorkflowActionParameters": {
|
| 51 |
+
"WFVariableName": "UserCommand",
|
| 52 |
+
},
|
| 53 |
+
},
|
| 54 |
+
# ── Action 3: Build JSON body ──
|
| 55 |
+
{
|
| 56 |
+
"WFWorkflowActionIdentifier": "is.workflow.actions.gettext",
|
| 57 |
+
"WFWorkflowActionParameters": {
|
| 58 |
+
"WFTextActionText": {
|
| 59 |
+
"Value": {
|
| 60 |
+
"attachmentsByRange": {
|
| 61 |
+
"{0, 1}": {
|
| 62 |
+
"Type": "Variable",
|
| 63 |
+
"VariableName": "UserCommand",
|
| 64 |
+
},
|
| 65 |
+
},
|
| 66 |
+
"string": '{"message": "\uFFFC"}',
|
| 67 |
+
},
|
| 68 |
+
"WFSerializationType": "WFTextTokenString",
|
| 69 |
+
},
|
| 70 |
+
},
|
| 71 |
+
},
|
| 72 |
+
# ── Action 4: POST to JARVIS server ──
|
| 73 |
{
|
| 74 |
"WFWorkflowActionIdentifier": "is.workflow.actions.downloadurl",
|
| 75 |
"WFWorkflowActionParameters": {
|
| 76 |
+
"WFURL": ASK_ENDPOINT,
|
| 77 |
+
"WFHTTPMethod": "POST",
|
| 78 |
+
"WFHTTPHeaders": {
|
| 79 |
+
"Value": {
|
| 80 |
+
"WFDictionaryFieldValueItems": [
|
| 81 |
+
{
|
| 82 |
+
"WFItemType": 0,
|
| 83 |
+
"WFKey": {"Value": {"string": "Content-Type"}, "WFSerializationType": "WFTextTokenString"},
|
| 84 |
+
"WFValue": {"Value": {"string": "application/json"}, "WFSerializationType": "WFTextTokenString"},
|
| 85 |
+
},
|
| 86 |
+
] + ([
|
| 87 |
+
{
|
| 88 |
+
"WFItemType": 0,
|
| 89 |
+
"WFKey": {"Value": {"string": "Authorization"}, "WFSerializationType": "WFTextTokenString"},
|
| 90 |
+
"WFValue": {"Value": {"string": f"Bearer {AUTH_TOKEN}"}, "WFSerializationType": "WFTextTokenString"},
|
| 91 |
+
},
|
| 92 |
+
] if AUTH_TOKEN else []),
|
| 93 |
+
},
|
| 94 |
+
"WFSerializationType": "WFDictionaryFieldValue",
|
| 95 |
+
},
|
| 96 |
+
"WFHTTPBodyType": "String",
|
| 97 |
+
"WFHTTPBodyString": {
|
| 98 |
+
"Value": {
|
| 99 |
+
"attachmentsByRange": {},
|
| 100 |
+
"string": "", # Will be set from previous action
|
| 101 |
+
},
|
| 102 |
+
"WFSerializationType": "WFTextTokenString",
|
| 103 |
+
},
|
| 104 |
+
},
|
| 105 |
+
},
|
| 106 |
+
# ── Action 5: Parse JSON response ──
|
| 107 |
+
{
|
| 108 |
+
"WFWorkflowActionIdentifier": "is.workflow.actions.detect.dictionary",
|
| 109 |
+
"WFWorkflowActionParameters": {},
|
| 110 |
+
},
|
| 111 |
+
# ── Action 6: Get "response" field from JSON ──
|
| 112 |
+
{
|
| 113 |
+
"WFWorkflowActionIdentifier": "is.workflow.actions.getvalueforkey",
|
| 114 |
+
"WFWorkflowActionParameters": {
|
| 115 |
+
"WFDictionaryKey": "response",
|
| 116 |
},
|
| 117 |
},
|
| 118 |
+
# ── Action 7: Speak the response aloud ──
|
| 119 |
+
{
|
| 120 |
+
"WFWorkflowActionIdentifier": "is.workflow.actions.speaktext",
|
| 121 |
+
"WFWorkflowActionParameters": {
|
| 122 |
+
"WFSpeakTextRate": 0.45, # Slightly faster
|
| 123 |
+
"WFSpeakTextWait": True,
|
| 124 |
+
},
|
| 125 |
+
},
|
| 126 |
+
# ── Action 8: Also show as notification ──
|
| 127 |
+
{
|
| 128 |
+
"WFWorkflowActionIdentifier": "is.workflow.actions.notification",
|
| 129 |
+
"WFWorkflowActionParameters": {
|
| 130 |
+
"WFNotificationActionTitle": "J.A.R.V.I.S.",
|
| 131 |
+
},
|
| 132 |
+
},
|
| 133 |
+
]
|
| 134 |
+
|
| 135 |
+
# Also try to trigger local Mac listener (best-effort, won't fail on mobile)
|
| 136 |
+
actions.append({
|
| 137 |
+
"WFWorkflowActionIdentifier": "is.workflow.actions.downloadurl",
|
| 138 |
+
"WFWorkflowActionParameters": {
|
| 139 |
+
"WFURL": LOCAL_TRIGGER_URL,
|
| 140 |
+
"WFHTTPMethod": "GET",
|
| 141 |
+
},
|
| 142 |
+
})
|
| 143 |
+
|
| 144 |
+
return {
|
| 145 |
+
"WFWorkflowMinimumClientVersionString": "900",
|
| 146 |
+
"WFWorkflowMinimumClientVersion": 900,
|
| 147 |
+
"WFWorkflowClientVersion": "2302.0.4",
|
| 148 |
+
"WFWorkflowHasShortcutInputVariables": False,
|
| 149 |
+
"WFWorkflowImportQuestions": [],
|
| 150 |
+
"WFWorkflowName": "JARVIS",
|
| 151 |
+
"WFWorkflowIcon": {
|
| 152 |
+
"WFWorkflowIconStartColor": 463140863, # Blue
|
| 153 |
+
"WFWorkflowIconGlyphNumber": 59511, # Microphone icon
|
| 154 |
+
},
|
| 155 |
+
"WFWorkflowTypes": [
|
| 156 |
+
"NCWidget", # Notification Center
|
| 157 |
+
"WatchKit", # Apple Watch
|
| 158 |
+
],
|
| 159 |
+
"WFWorkflowInputContentItemClasses": [
|
| 160 |
+
"WFStringContentItem",
|
| 161 |
+
],
|
| 162 |
+
"WFWorkflowActions": actions,
|
| 163 |
+
}
|
| 164 |
+
|
| 165 |
+
|
| 166 |
+
def sign_and_install(shortcut_data: dict, name: str):
|
| 167 |
+
"""Sign and install a shortcut."""
|
| 168 |
+
unsigned_path = os.path.join(JARVIS_DIR, f"{name}-unsigned.shortcut")
|
| 169 |
+
final_path = os.path.join(JARVIS_DIR, f"{name}.shortcut")
|
| 170 |
+
|
| 171 |
+
with open(unsigned_path, "wb") as f:
|
| 172 |
+
plistlib.dump(shortcut_data, f, fmt=plistlib.FMT_BINARY)
|
| 173 |
+
|
| 174 |
+
# Sign the shortcut
|
| 175 |
+
try:
|
| 176 |
+
subprocess.run(
|
| 177 |
+
["shortcuts", "sign", "-m", "people-who-know-me",
|
| 178 |
+
"-i", unsigned_path, "-o", final_path],
|
| 179 |
+
check=True, capture_output=True,
|
| 180 |
+
)
|
| 181 |
+
os.remove(unsigned_path)
|
| 182 |
+
print(f" ✓ Signed: {final_path}")
|
| 183 |
+
except Exception as e:
|
| 184 |
+
print(f" [!] Signing failed: {e}")
|
| 185 |
+
os.rename(unsigned_path, final_path)
|
| 186 |
+
print(f" ✓ Unsigned: {final_path}")
|
| 187 |
+
|
| 188 |
+
# Auto-import
|
| 189 |
+
try:
|
| 190 |
+
result = subprocess.run(
|
| 191 |
+
["open", final_path],
|
| 192 |
+
capture_output=True, text=True, timeout=10,
|
| 193 |
+
)
|
| 194 |
+
if result.returncode == 0:
|
| 195 |
+
print(f" → Shortcuts app should open — click 'Add Shortcut'")
|
| 196 |
+
else:
|
| 197 |
+
print(f" → Manual import: open '{final_path}'")
|
| 198 |
+
except Exception as e:
|
| 199 |
+
print(f" → Auto-import error: {e}")
|
| 200 |
+
|
| 201 |
+
|
| 202 |
+
if __name__ == "__main__":
|
| 203 |
+
print(f"JARVIS Server: {JARVIS_API_URL}")
|
| 204 |
+
print(f"Ask Endpoint: {ASK_ENDPOINT}")
|
| 205 |
+
if AUTH_TOKEN:
|
| 206 |
+
print(f"Auth Token: {AUTH_TOKEN[:8]}...")
|
| 207 |
+
print()
|
| 208 |
+
|
| 209 |
+
print("[1/1] Building JARVIS Siri Shortcut...")
|
| 210 |
+
shortcut = build_jarvis_shortcut()
|
| 211 |
+
sign_and_install(shortcut, "JARVIS")
|
| 212 |
+
|
| 213 |
+
print()
|
| 214 |
+
print("╔═══════════════════════════════════════════╗")
|
| 215 |
+
print("║ JARVIS Works On All Apple Devices: ║")
|
| 216 |
+
print("║ ║")
|
| 217 |
+
print('║ • Mac/iPhone/iPad: "Hey Siri, JARVIS" ║')
|
| 218 |
+
print("║ • Apple Watch: Raise wrist → 'JARVIS' ║")
|
| 219 |
+
print("║ • iPhone/iPad: PWA (Add to Home Screen) ║")
|
| 220 |
+
print("║ • Any device: Telegram bot ║")
|
| 221 |
+
print("║ ║")
|
| 222 |
+
print("║ To sync shortcut to iPhone/iPad/Watch: ║")
|
| 223 |
+
print("║ It syncs automatically via iCloud. ║")
|
| 224 |
+
print("║ Or AirDrop the .shortcut file. ║")
|
| 225 |
+
print("╚═══════════════════════════════════════════╝")
|
|
@@ -1,7 +1,8 @@
|
|
| 1 |
"""
|
| 2 |
J.A.R.V.I.S. — macOS Native App with WebView UI.
|
| 3 |
|
| 4 |
-
|
|
|
|
| 5 |
No terminal. No browser. Just JARVIS.
|
| 6 |
"""
|
| 7 |
|
|
@@ -31,10 +32,31 @@ from AppKit import (
|
|
| 31 |
from Foundation import NSObject, NSURL, NSURLRequest, NSMakeRect
|
| 32 |
from WebKit import WKWebView, WKWebViewConfiguration, WKUserContentController
|
| 33 |
|
| 34 |
-
|
|
|
|
|
|
|
| 35 |
JARVIS_DIR = os.path.dirname(os.path.abspath(__file__))
|
| 36 |
|
| 37 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 38 |
class AppDelegate(NSObject):
|
| 39 |
"""Main application delegate — creates the WebView window."""
|
| 40 |
|
|
@@ -94,17 +116,40 @@ class AppDelegate(NSObject):
|
|
| 94 |
NSApp.activateIgnoringOtherApps_(True)
|
| 95 |
|
| 96 |
def _load_when_ready(self):
|
| 97 |
-
"""
|
|
|
|
| 98 |
import httpx
|
| 99 |
|
| 100 |
-
|
|
|
|
| 101 |
try:
|
| 102 |
-
httpx.get(f"{
|
|
|
|
| 103 |
break
|
| 104 |
except Exception:
|
| 105 |
time.sleep(1)
|
| 106 |
|
| 107 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 108 |
request = NSURLRequest.requestWithURL_(url)
|
| 109 |
self.webview.performSelectorOnMainThread_withObject_waitUntilDone_(
|
| 110 |
objc.selector(self.webview.loadRequest_, signature=b"v@:@"),
|
|
|
|
| 1 |
"""
|
| 2 |
J.A.R.V.I.S. — macOS Native App with WebView UI.
|
| 3 |
|
| 4 |
+
Loads the JARVIS interface directly in a native WebKit window.
|
| 5 |
+
Tries local server first, then falls back to HuggingFace Space / Render.
|
| 6 |
No terminal. No browser. Just JARVIS.
|
| 7 |
"""
|
| 8 |
|
|
|
|
| 32 |
from Foundation import NSObject, NSURL, NSURLRequest, NSMakeRect
|
| 33 |
from WebKit import WKWebView, WKWebViewConfiguration, WKUserContentController
|
| 34 |
|
| 35 |
+
JARVIS_LOCAL_URL = os.getenv("JARVIS_URL", "http://localhost:8000")
|
| 36 |
+
HF_SPACE_URL = os.getenv("HF_SPACE_URL", "https://v1deh-jarvis.hf.space")
|
| 37 |
+
RENDER_URL = os.getenv("RENDER_URL", "")
|
| 38 |
JARVIS_DIR = os.path.dirname(os.path.abspath(__file__))
|
| 39 |
|
| 40 |
|
| 41 |
+
def _find_server() -> str:
|
| 42 |
+
"""Try local → HF Space → Render. Return first reachable URL."""
|
| 43 |
+
import httpx
|
| 44 |
+
candidates = [JARVIS_LOCAL_URL, HF_SPACE_URL]
|
| 45 |
+
if RENDER_URL:
|
| 46 |
+
candidates.append(RENDER_URL)
|
| 47 |
+
|
| 48 |
+
for url in candidates:
|
| 49 |
+
if not url:
|
| 50 |
+
continue
|
| 51 |
+
try:
|
| 52 |
+
resp = httpx.get(f"{url}/api/status", timeout=5)
|
| 53 |
+
if resp.status_code == 200:
|
| 54 |
+
return url
|
| 55 |
+
except Exception:
|
| 56 |
+
continue
|
| 57 |
+
return ""
|
| 58 |
+
|
| 59 |
+
|
| 60 |
class AppDelegate(NSObject):
|
| 61 |
"""Main application delegate — creates the WebView window."""
|
| 62 |
|
|
|
|
| 116 |
NSApp.activateIgnoringOtherApps_(True)
|
| 117 |
|
| 118 |
def _load_when_ready(self):
|
| 119 |
+
"""Find reachable server (local or HF Space) and load the UI."""
|
| 120 |
+
# Try up to 15s for local server (it may be starting), then check remotes
|
| 121 |
import httpx
|
| 122 |
|
| 123 |
+
target_url = ""
|
| 124 |
+
for i in range(15):
|
| 125 |
try:
|
| 126 |
+
httpx.get(f"{JARVIS_LOCAL_URL}/api/status", timeout=3)
|
| 127 |
+
target_url = JARVIS_LOCAL_URL
|
| 128 |
break
|
| 129 |
except Exception:
|
| 130 |
time.sleep(1)
|
| 131 |
|
| 132 |
+
# Local server not up — try HF Space / Render directly
|
| 133 |
+
if not target_url:
|
| 134 |
+
target_url = _find_server()
|
| 135 |
+
|
| 136 |
+
if not target_url:
|
| 137 |
+
# Nothing reachable — show error page in WebView
|
| 138 |
+
error_html = (
|
| 139 |
+
"data:text/html,"
|
| 140 |
+
"<html><body style='background:#0a0e17;color:#e0e6ed;font-family:monospace;"
|
| 141 |
+
"display:flex;align-items:center;justify-content:center;height:100vh;"
|
| 142 |
+
"flex-direction:column;'>"
|
| 143 |
+
"<h1 style='color:#00b4d8;letter-spacing:4px;'>J.A.R.V.I.S.</h1>"
|
| 144 |
+
"<p style='color:#8899aa;margin-top:16px;'>Cannot reach any server.</p>"
|
| 145 |
+
"<p style='color:#8899aa;font-size:12px;margin-top:8px;'>"
|
| 146 |
+
"Start local: <code>python server.py</code><br>"
|
| 147 |
+
"Or set HF_SPACE_URL / RENDER_URL in .env</p>"
|
| 148 |
+
"</body></html>"
|
| 149 |
+
)
|
| 150 |
+
target_url = error_html
|
| 151 |
+
|
| 152 |
+
url = NSURL.URLWithString_(target_url)
|
| 153 |
request = NSURLRequest.requestWithURL_(url)
|
| 154 |
self.webview.performSelectorOnMainThread_withObject_waitUntilDone_(
|
| 155 |
objc.selector(self.webview.loadRequest_, signature=b"v@:@"),
|
|
@@ -84,6 +84,81 @@ def touch_activity():
|
|
| 84 |
_last_activity = time.time()
|
| 85 |
|
| 86 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 87 |
def check_mic_permission() -> bool:
|
| 88 |
"""Test if microphone is actually returning audio data.
|
| 89 |
macOS silently returns zeros when permission is not granted."""
|
|
@@ -495,6 +570,9 @@ EXIT_PHRASES = {"goodbye", "that's all", "thats all", "thank you", "thanks",
|
|
| 495 |
"go to sleep", "nevermind", "never mind", "bye", "dismiss"}
|
| 496 |
CONVERSATION_TIMEOUT = 30 # seconds of silence before auto-ending session
|
| 497 |
|
|
|
|
|
|
|
|
|
|
| 498 |
|
| 499 |
def handle_activation(command_text=None):
|
| 500 |
"""
|
|
@@ -508,6 +586,8 @@ def handle_activation(command_text=None):
|
|
| 508 |
return
|
| 509 |
|
| 510 |
touch_activity()
|
|
|
|
|
|
|
| 511 |
|
| 512 |
try:
|
| 513 |
wake_pause_event.set()
|
|
@@ -532,8 +612,9 @@ def handle_activation(command_text=None):
|
|
| 532 |
|
| 533 |
# Check if user wants to end conversation
|
| 534 |
if command_text.lower().strip().rstrip(".!") in EXIT_PHRASES:
|
| 535 |
-
speak("Standing by, sir.")
|
| 536 |
-
log.info("Conversation ended by user")
|
|
|
|
| 537 |
break
|
| 538 |
|
| 539 |
# Process the command
|
|
@@ -543,6 +624,12 @@ def handle_activation(command_text=None):
|
|
| 543 |
speak(answer)
|
| 544 |
|
| 545 |
# Listen for follow-up (with shorter timeout)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 546 |
log.info("Listening for follow-up...")
|
| 547 |
command_text = _listen_for_command()
|
| 548 |
if not command_text:
|
|
@@ -643,6 +730,16 @@ def start_local_wake_listener():
|
|
| 643 |
diag_max_amp = 0
|
| 644 |
|
| 645 |
while True:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 646 |
if wake_pause_event.is_set():
|
| 647 |
close_stream()
|
| 648 |
clear_audio_state()
|
|
@@ -890,12 +987,10 @@ def server_watchdog():
|
|
| 890 |
_device_id = None
|
| 891 |
|
| 892 |
|
| 893 |
-
def
|
| 894 |
-
"""
|
| 895 |
global _device_id
|
| 896 |
device_id_file = os.path.join(JARVIS_DIR, ".device_id")
|
| 897 |
-
|
| 898 |
-
# Load or generate device ID
|
| 899 |
if os.path.exists(device_id_file):
|
| 900 |
with open(device_id_file) as f:
|
| 901 |
_device_id = f.read().strip()
|
|
@@ -904,12 +999,17 @@ def device_heartbeat_loop():
|
|
| 904 |
_device_id = str(uuid.uuid4())[:12]
|
| 905 |
with open(device_id_file, "w") as f:
|
| 906 |
f.write(_device_id)
|
|
|
|
|
|
|
| 907 |
|
|
|
|
|
|
|
|
|
|
| 908 |
log.info(f"Device heartbeat: ID={_device_id}, interval=60s")
|
| 909 |
while True:
|
| 910 |
try:
|
| 911 |
httpx.post(
|
| 912 |
-
f"{JARVIS_URL}/api/
|
| 913 |
json={"device_id": _device_id},
|
| 914 |
timeout=5,
|
| 915 |
)
|
|
@@ -918,6 +1018,96 @@ def device_heartbeat_loop():
|
|
| 918 |
time.sleep(60)
|
| 919 |
|
| 920 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 921 |
# ─── Clap/Snap Detection ─────────────────────────
|
| 922 |
def start_clap_detector():
|
| 923 |
"""Start clap/snap detection as an additional wake trigger."""
|
|
@@ -975,17 +1165,44 @@ def run_menubar():
|
|
| 975 |
self.title = "J"
|
| 976 |
self.menu = [
|
| 977 |
rumps.MenuItem("Activate JARVIS", callback=self.on_activate),
|
|
|
|
| 978 |
None,
|
| 979 |
rumps.MenuItem("Status", callback=self.on_status),
|
| 980 |
None,
|
| 981 |
rumps.MenuItem("Quit", callback=self.on_quit),
|
| 982 |
]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 983 |
|
| 984 |
def on_activate(self, _):
|
| 985 |
touch_activity()
|
|
|
|
| 986 |
threading.Thread(target=handle_activation, daemon=True).start()
|
| 987 |
rumps.notification("J.A.R.V.I.S.", "", "Listening...", sound=False)
|
| 988 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 989 |
def on_status(self, _):
|
| 990 |
threading.Thread(target=self._check_status, daemon=True).start()
|
| 991 |
|
|
@@ -998,6 +1215,7 @@ def run_menubar():
|
|
| 998 |
parts.append("Server: offline")
|
| 999 |
parts.append(f"Wake: {ACTIVE_WAKE_MODE}")
|
| 1000 |
parts.append(f"Clap: {'on' if ENABLE_CLAP_DETECTION else 'off'}")
|
|
|
|
| 1001 |
rumps.notification("J.A.R.V.I.S. Status", "", " | ".join(parts))
|
| 1002 |
|
| 1003 |
def on_quit(self, _):
|
|
@@ -1045,12 +1263,18 @@ def main():
|
|
| 1045 |
threading.Thread(target=start_hotkey_listener, daemon=True).start()
|
| 1046 |
threading.Thread(target=server_watchdog, daemon=True).start()
|
| 1047 |
threading.Thread(target=device_heartbeat_loop, daemon=True).start()
|
|
|
|
| 1048 |
|
| 1049 |
# Check mic permission before starting wake word listeners
|
| 1050 |
mic_ok = check_mic_permission()
|
| 1051 |
if not mic_ok:
|
| 1052 |
log.warning("Mic check returned silent — will attempt wake listeners anyway (TCC may grant access to .app bundle)")
|
| 1053 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1054 |
if ACTIVE_WAKE_MODE == "local":
|
| 1055 |
threading.Thread(target=start_local_wake_listener, daemon=True).start()
|
| 1056 |
|
|
|
|
| 84 |
_last_activity = time.time()
|
| 85 |
|
| 86 |
|
| 87 |
+
# ─── Cached automation permission state ──────────
|
| 88 |
+
_automation_granted: dict[str, bool] = {}
|
| 89 |
+
|
| 90 |
+
|
| 91 |
+
def check_automation_permission(app_name: str) -> bool:
|
| 92 |
+
"""Check if JARVIS has Automation permission for a given app.
|
| 93 |
+
Caches the result so macOS only prompts once per app per session."""
|
| 94 |
+
if app_name in _automation_granted:
|
| 95 |
+
return _automation_granted[app_name]
|
| 96 |
+
|
| 97 |
+
try:
|
| 98 |
+
result = subprocess.run(
|
| 99 |
+
["osascript", "-e", f'tell application "{app_name}" to name'],
|
| 100 |
+
capture_output=True, text=True, timeout=10,
|
| 101 |
+
)
|
| 102 |
+
granted = result.returncode == 0
|
| 103 |
+
_automation_granted[app_name] = granted
|
| 104 |
+
if not granted:
|
| 105 |
+
log.warning(
|
| 106 |
+
f"Automation permission denied for {app_name}. "
|
| 107 |
+
f"Fix: System Settings → Privacy & Security → Automation → enable JARVIS for {app_name}"
|
| 108 |
+
)
|
| 109 |
+
return granted
|
| 110 |
+
except Exception as e:
|
| 111 |
+
log.warning(f"Automation permission check for {app_name} failed: {e}")
|
| 112 |
+
return False
|
| 113 |
+
|
| 114 |
+
|
| 115 |
+
def request_automation_permissions():
|
| 116 |
+
"""Request Automation permissions for key apps at startup.
|
| 117 |
+
macOS will show a one-time prompt per app if not already granted.
|
| 118 |
+
This must run from the JARVIS process (not Terminal) so permissions
|
| 119 |
+
are associated with the JARVIS app bundle."""
|
| 120 |
+
apps_to_request = [
|
| 121 |
+
"System Events", # Required for UI automation, keystrokes, app control
|
| 122 |
+
"Finder", # File operations
|
| 123 |
+
"Calendar", # Calendar events
|
| 124 |
+
"Notes", # Note creation
|
| 125 |
+
"Reminders", # Reminders
|
| 126 |
+
"Mail", # Email
|
| 127 |
+
]
|
| 128 |
+
|
| 129 |
+
log.info("Requesting Automation permissions for key apps...")
|
| 130 |
+
for app in apps_to_request:
|
| 131 |
+
granted = check_automation_permission(app)
|
| 132 |
+
if granted:
|
| 133 |
+
log.info(f" ✓ Automation: {app}")
|
| 134 |
+
else:
|
| 135 |
+
log.warning(f" ✗ Automation: {app} — user must grant in System Settings")
|
| 136 |
+
notify(
|
| 137 |
+
"JARVIS — Permission Needed",
|
| 138 |
+
f"Grant Automation access for {app}: System Settings → Privacy & Security → Automation",
|
| 139 |
+
)
|
| 140 |
+
# Small delay between prompts so the user sees each one
|
| 141 |
+
time.sleep(1)
|
| 142 |
+
|
| 143 |
+
# Check Accessibility (needed for global hotkey and UI control)
|
| 144 |
+
try:
|
| 145 |
+
result = subprocess.run(
|
| 146 |
+
["osascript", "-e",
|
| 147 |
+
'tell application "System Events" to key code 0'],
|
| 148 |
+
capture_output=True, text=True, timeout=5,
|
| 149 |
+
)
|
| 150 |
+
if result.returncode != 0:
|
| 151 |
+
log.warning("Accessibility access not granted.")
|
| 152 |
+
notify(
|
| 153 |
+
"JARVIS — Accessibility Needed",
|
| 154 |
+
"System Settings → Privacy & Security → Accessibility → enable JARVIS",
|
| 155 |
+
)
|
| 156 |
+
except Exception:
|
| 157 |
+
pass
|
| 158 |
+
|
| 159 |
+
log.info("Automation permission requests complete")
|
| 160 |
+
|
| 161 |
+
|
| 162 |
def check_mic_permission() -> bool:
|
| 163 |
"""Test if microphone is actually returning audio data.
|
| 164 |
macOS silently returns zeros when permission is not granted."""
|
|
|
|
| 570 |
"go to sleep", "nevermind", "never mind", "bye", "dismiss"}
|
| 571 |
CONVERSATION_TIMEOUT = 30 # seconds of silence before auto-ending session
|
| 572 |
|
| 573 |
+
# Standby mode — mic is fully off, only hotkey/Siri/menubar can re-activate
|
| 574 |
+
standby_mode = threading.Event()
|
| 575 |
+
|
| 576 |
|
| 577 |
def handle_activation(command_text=None):
|
| 578 |
"""
|
|
|
|
| 586 |
return
|
| 587 |
|
| 588 |
touch_activity()
|
| 589 |
+
# Exit standby when user explicitly activates
|
| 590 |
+
standby_mode.clear()
|
| 591 |
|
| 592 |
try:
|
| 593 |
wake_pause_event.set()
|
|
|
|
| 612 |
|
| 613 |
# Check if user wants to end conversation
|
| 614 |
if command_text.lower().strip().rstrip(".!") in EXIT_PHRASES:
|
| 615 |
+
speak("Standing by, sir. Mic is off.")
|
| 616 |
+
log.info("Conversation ended by user — entering standby (mic off)")
|
| 617 |
+
standby_mode.set()
|
| 618 |
break
|
| 619 |
|
| 620 |
# Process the command
|
|
|
|
| 624 |
speak(answer)
|
| 625 |
|
| 626 |
# Listen for follow-up (with shorter timeout)
|
| 627 |
+
# If server returned an error, prompt user to retry
|
| 628 |
+
is_error = answer.startswith(("I can't reach", "The request timed out",
|
| 629 |
+
"I got a garbled", "I'm having trouble"))
|
| 630 |
+
if is_error:
|
| 631 |
+
speak("Would you like to try again, sir?")
|
| 632 |
+
|
| 633 |
log.info("Listening for follow-up...")
|
| 634 |
command_text = _listen_for_command()
|
| 635 |
if not command_text:
|
|
|
|
| 730 |
diag_max_amp = 0
|
| 731 |
|
| 732 |
while True:
|
| 733 |
+
# Standby mode — mic fully off until hotkey/Siri/menubar reactivates
|
| 734 |
+
if standby_mode.is_set():
|
| 735 |
+
close_stream()
|
| 736 |
+
clear_audio_state()
|
| 737 |
+
diag_frame_count = 0
|
| 738 |
+
diag_speech_count = 0
|
| 739 |
+
diag_max_amp = 0
|
| 740 |
+
time.sleep(0.2)
|
| 741 |
+
continue
|
| 742 |
+
|
| 743 |
if wake_pause_event.is_set():
|
| 744 |
close_stream()
|
| 745 |
clear_audio_state()
|
|
|
|
| 987 |
_device_id = None
|
| 988 |
|
| 989 |
|
| 990 |
+
def _load_device_id():
|
| 991 |
+
"""Load or generate a persistent device ID."""
|
| 992 |
global _device_id
|
| 993 |
device_id_file = os.path.join(JARVIS_DIR, ".device_id")
|
|
|
|
|
|
|
| 994 |
if os.path.exists(device_id_file):
|
| 995 |
with open(device_id_file) as f:
|
| 996 |
_device_id = f.read().strip()
|
|
|
|
| 999 |
_device_id = str(uuid.uuid4())[:12]
|
| 1000 |
with open(device_id_file, "w") as f:
|
| 1001 |
f.write(_device_id)
|
| 1002 |
+
return _device_id
|
| 1003 |
+
|
| 1004 |
|
| 1005 |
+
def device_heartbeat_loop():
|
| 1006 |
+
"""Send periodic heartbeats to the server so the device shows as online."""
|
| 1007 |
+
_load_device_id()
|
| 1008 |
log.info(f"Device heartbeat: ID={_device_id}, interval=60s")
|
| 1009 |
while True:
|
| 1010 |
try:
|
| 1011 |
httpx.post(
|
| 1012 |
+
f"{JARVIS_URL}/api/devices/heartbeat",
|
| 1013 |
json={"device_id": _device_id},
|
| 1014 |
timeout=5,
|
| 1015 |
)
|
|
|
|
| 1018 |
time.sleep(60)
|
| 1019 |
|
| 1020 |
|
| 1021 |
+
def device_websocket_loop():
|
| 1022 |
+
"""Connect to server via WebSocket to receive delegated tool commands in real-time.
|
| 1023 |
+
|
| 1024 |
+
When the JARVIS server runs on HF Space (Linux), macOS-only tools like
|
| 1025 |
+
open_app, set_volume, spotify_play, etc. get delegated to this connected
|
| 1026 |
+
device for local execution.
|
| 1027 |
+
"""
|
| 1028 |
+
import websocket as ws_client
|
| 1029 |
+
|
| 1030 |
+
_load_device_id()
|
| 1031 |
+
ws_url = JARVIS_URL.replace("http://", "ws://").replace("https://", "wss://")
|
| 1032 |
+
endpoint = f"{ws_url}/ws/device/{_device_id}"
|
| 1033 |
+
|
| 1034 |
+
def _execute_tool_locally(tool_name: str, tool_args: dict) -> str:
|
| 1035 |
+
"""Execute a tool on this Mac and return the result."""
|
| 1036 |
+
try:
|
| 1037 |
+
# Import tool registry (already loaded by server.py imports,
|
| 1038 |
+
# but listener runs standalone so we import directly)
|
| 1039 |
+
sys.path.insert(0, JARVIS_DIR)
|
| 1040 |
+
from tools import TOOL_REGISTRY, execute_tool as _exec
|
| 1041 |
+
import tools.builtin
|
| 1042 |
+
import tools.system_control
|
| 1043 |
+
import tools.app_automation
|
| 1044 |
+
import tools.vscode_tools
|
| 1045 |
+
import tools.device_control
|
| 1046 |
+
|
| 1047 |
+
if tool_name not in TOOL_REGISTRY:
|
| 1048 |
+
return f"Error: Unknown tool '{tool_name}'"
|
| 1049 |
+
|
| 1050 |
+
func = TOOL_REGISTRY[tool_name]["function"]
|
| 1051 |
+
result = func(**tool_args)
|
| 1052 |
+
# Handle async tools
|
| 1053 |
+
if hasattr(result, "__await__"):
|
| 1054 |
+
import asyncio
|
| 1055 |
+
loop = asyncio.new_event_loop()
|
| 1056 |
+
result = loop.run_until_complete(result)
|
| 1057 |
+
loop.close()
|
| 1058 |
+
return str(result)
|
| 1059 |
+
except Exception as e:
|
| 1060 |
+
log.error(f"Local tool execution failed: {tool_name}({tool_args}) -> {e}")
|
| 1061 |
+
return f"Error: {e}"
|
| 1062 |
+
|
| 1063 |
+
while True:
|
| 1064 |
+
try:
|
| 1065 |
+
log.info(f"Device WS: connecting to {endpoint}")
|
| 1066 |
+
ws = ws_client.WebSocket()
|
| 1067 |
+
ws.connect(endpoint, timeout=10)
|
| 1068 |
+
log.info(f"Device WS: connected as {_device_id}")
|
| 1069 |
+
|
| 1070 |
+
while True:
|
| 1071 |
+
msg = ws.recv()
|
| 1072 |
+
if not msg:
|
| 1073 |
+
continue
|
| 1074 |
+
data = json.loads(msg)
|
| 1075 |
+
|
| 1076 |
+
if data.get("type") == "command" and data.get("action") == "execute_tool":
|
| 1077 |
+
tool_name = data.get("tool", "")
|
| 1078 |
+
tool_args = data.get("args", {})
|
| 1079 |
+
log.info(f"Device WS: executing delegated tool {tool_name}({tool_args})")
|
| 1080 |
+
notify("JARVIS — Remote Command", f"Executing: {tool_name}")
|
| 1081 |
+
|
| 1082 |
+
result = _execute_tool_locally(tool_name, tool_args)
|
| 1083 |
+
log.info(f"Device WS: {tool_name} result: {result[:200]}")
|
| 1084 |
+
|
| 1085 |
+
# Report result back to server
|
| 1086 |
+
cmd_id = data.get("cmd_id", "")
|
| 1087 |
+
if cmd_id:
|
| 1088 |
+
ws.send(json.dumps({
|
| 1089 |
+
"type": "command_result",
|
| 1090 |
+
"cmd_id": cmd_id,
|
| 1091 |
+
"result": result,
|
| 1092 |
+
}))
|
| 1093 |
+
notify("JARVIS", f"{tool_name}: {result[:100]}")
|
| 1094 |
+
|
| 1095 |
+
elif data.get("type") == "command":
|
| 1096 |
+
# Generic command (not a tool call) — speak it
|
| 1097 |
+
command_text = data.get("command", "")
|
| 1098 |
+
if command_text:
|
| 1099 |
+
log.info(f"Device WS: received command: {command_text}")
|
| 1100 |
+
threading.Thread(
|
| 1101 |
+
target=handle_activation,
|
| 1102 |
+
kwargs={"command_text": command_text},
|
| 1103 |
+
daemon=True,
|
| 1104 |
+
).start()
|
| 1105 |
+
|
| 1106 |
+
except Exception as e:
|
| 1107 |
+
log.warning(f"Device WS: connection lost ({e}), reconnecting in 10s...")
|
| 1108 |
+
time.sleep(10)
|
| 1109 |
+
|
| 1110 |
+
|
| 1111 |
# ─── Clap/Snap Detection ─────────────────────────
|
| 1112 |
def start_clap_detector():
|
| 1113 |
"""Start clap/snap detection as an additional wake trigger."""
|
|
|
|
| 1165 |
self.title = "J"
|
| 1166 |
self.menu = [
|
| 1167 |
rumps.MenuItem("Activate JARVIS", callback=self.on_activate),
|
| 1168 |
+
rumps.MenuItem("Resume Listening", callback=self.on_resume),
|
| 1169 |
None,
|
| 1170 |
rumps.MenuItem("Status", callback=self.on_status),
|
| 1171 |
None,
|
| 1172 |
rumps.MenuItem("Quit", callback=self.on_quit),
|
| 1173 |
]
|
| 1174 |
+
# Monitor standby mode to update menu bar title
|
| 1175 |
+
threading.Thread(target=self._watch_standby, daemon=True).start()
|
| 1176 |
+
|
| 1177 |
+
def _watch_standby(self):
|
| 1178 |
+
"""Update menu bar icon when entering/leaving standby."""
|
| 1179 |
+
was_standby = False
|
| 1180 |
+
while True:
|
| 1181 |
+
is_standby = standby_mode.is_set()
|
| 1182 |
+
if is_standby and not was_standby:
|
| 1183 |
+
self.title = "J💤"
|
| 1184 |
+
log.info("Menu bar: standby mode (mic off)")
|
| 1185 |
+
elif not is_standby and was_standby:
|
| 1186 |
+
self.title = "J"
|
| 1187 |
+
log.info("Menu bar: active mode")
|
| 1188 |
+
was_standby = is_standby
|
| 1189 |
+
time.sleep(1)
|
| 1190 |
|
| 1191 |
def on_activate(self, _):
|
| 1192 |
touch_activity()
|
| 1193 |
+
standby_mode.clear() # Exit standby on manual activation
|
| 1194 |
threading.Thread(target=handle_activation, daemon=True).start()
|
| 1195 |
rumps.notification("J.A.R.V.I.S.", "", "Listening...", sound=False)
|
| 1196 |
|
| 1197 |
+
def on_resume(self, _):
|
| 1198 |
+
"""Resume wake word listening after standby."""
|
| 1199 |
+
if standby_mode.is_set():
|
| 1200 |
+
standby_mode.clear()
|
| 1201 |
+
rumps.notification("J.A.R.V.I.S.", "", "Wake word listening resumed", sound=True)
|
| 1202 |
+
speak("I'm back online, sir.")
|
| 1203 |
+
else:
|
| 1204 |
+
rumps.notification("J.A.R.V.I.S.", "", "Already listening", sound=False)
|
| 1205 |
+
|
| 1206 |
def on_status(self, _):
|
| 1207 |
threading.Thread(target=self._check_status, daemon=True).start()
|
| 1208 |
|
|
|
|
| 1215 |
parts.append("Server: offline")
|
| 1216 |
parts.append(f"Wake: {ACTIVE_WAKE_MODE}")
|
| 1217 |
parts.append(f"Clap: {'on' if ENABLE_CLAP_DETECTION else 'off'}")
|
| 1218 |
+
parts.append(f"Mic: {'off (standby)' if standby_mode.is_set() else 'active'}")
|
| 1219 |
rumps.notification("J.A.R.V.I.S. Status", "", " | ".join(parts))
|
| 1220 |
|
| 1221 |
def on_quit(self, _):
|
|
|
|
| 1263 |
threading.Thread(target=start_hotkey_listener, daemon=True).start()
|
| 1264 |
threading.Thread(target=server_watchdog, daemon=True).start()
|
| 1265 |
threading.Thread(target=device_heartbeat_loop, daemon=True).start()
|
| 1266 |
+
threading.Thread(target=device_websocket_loop, daemon=True).start()
|
| 1267 |
|
| 1268 |
# Check mic permission before starting wake word listeners
|
| 1269 |
mic_ok = check_mic_permission()
|
| 1270 |
if not mic_ok:
|
| 1271 |
log.warning("Mic check returned silent — will attempt wake listeners anyway (TCC may grant access to .app bundle)")
|
| 1272 |
|
| 1273 |
+
# Request Automation/Accessibility permissions at startup
|
| 1274 |
+
# This triggers macOS permission prompts from the JARVIS process
|
| 1275 |
+
# so permissions are associated with the JARVIS app bundle
|
| 1276 |
+
threading.Thread(target=request_automation_permissions, daemon=True).start()
|
| 1277 |
+
|
| 1278 |
if ACTIVE_WAKE_MODE == "local":
|
| 1279 |
threading.Thread(target=start_local_wake_listener, daemon=True).start()
|
| 1280 |
|
|
@@ -33,7 +33,7 @@ load_dotenv()
|
|
| 33 |
from memory import Memory
|
| 34 |
from llm import SYSTEM_PROMPT, stream_response, get_active_backend, get_available_backends, FREE_MODELS, HF_FREE_MODELS
|
| 35 |
from stm import apply_stm, AutoTune
|
| 36 |
-
from tools import get_tools_prompt, execute_tool, TOOL_REGISTRY
|
| 37 |
import tools.builtin # Register built-in tools
|
| 38 |
import tools.system_control # System control tools
|
| 39 |
import tools.user_tools # User profile, routines, work sessions
|
|
@@ -43,6 +43,9 @@ import tools.app_automation # Deep app control (Spotify, Notes, Calendar, Mail,
|
|
| 43 |
import tools.device_onboarding # User device registration & cross-device commands
|
| 44 |
from user_profile import get_user_context, get_preferences, get_today_routine, get_work_history, get_active_work_session
|
| 45 |
|
|
|
|
|
|
|
|
|
|
| 46 |
# ── Auth Token ────────────────────────────────────────────────────
|
| 47 |
# Set JARVIS_AUTH_TOKEN in .env for a fixed token; otherwise a random one is
|
| 48 |
# generated each startup and printed to the console.
|
|
|
|
| 33 |
from memory import Memory
|
| 34 |
from llm import SYSTEM_PROMPT, stream_response, get_active_backend, get_available_backends, FREE_MODELS, HF_FREE_MODELS
|
| 35 |
from stm import apply_stm, AutoTune
|
| 36 |
+
from tools import get_tools_prompt, execute_tool, TOOL_REGISTRY, register_macos_tools
|
| 37 |
import tools.builtin # Register built-in tools
|
| 38 |
import tools.system_control # System control tools
|
| 39 |
import tools.user_tools # User profile, routines, work sessions
|
|
|
|
| 43 |
import tools.device_onboarding # User device registration & cross-device commands
|
| 44 |
from user_profile import get_user_context, get_preferences, get_today_routine, get_work_history, get_active_work_session
|
| 45 |
|
| 46 |
+
# Tag macOS-only tools so they get delegated to connected devices on Linux (HF Space)
|
| 47 |
+
register_macos_tools()
|
| 48 |
+
|
| 49 |
# ── Auth Token ────────────────────────────────────────────────────
|
| 50 |
# Set JARVIS_AUTH_TOKEN in .env for a fixed token; otherwise a random one is
|
| 51 |
# generated each startup and printed to the console.
|
|
|
|
|
|
|
|
@@ -2,14 +2,21 @@
|
|
| 2 |
<html lang="en">
|
| 3 |
<head>
|
| 4 |
<meta charset="UTF-8">
|
| 5 |
-
<meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=no">
|
| 6 |
<meta name="apple-mobile-web-app-capable" content="yes">
|
| 7 |
<meta name="apple-mobile-web-app-status-bar-style" content="black-translucent">
|
| 8 |
<meta name="apple-mobile-web-app-title" content="JARVIS">
|
| 9 |
<meta name="mobile-web-app-capable" content="yes">
|
| 10 |
<meta name="theme-color" content="#0a0e17">
|
|
|
|
| 11 |
<link rel="manifest" href="/static/manifest.json">
|
| 12 |
-
<
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 13 |
<title>J.A.R.V.I.S.</title>
|
| 14 |
<style>
|
| 15 |
* { margin: 0; padding: 0; box-sizing: border-box; }
|
|
@@ -482,6 +489,22 @@
|
|
| 482 |
.welcome h2 { font-size: 20px; }
|
| 483 |
.quick-actions { gap: 6px; }
|
| 484 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 485 |
</style>
|
| 486 |
</head>
|
| 487 |
<body>
|
|
@@ -1159,28 +1182,41 @@
|
|
| 1159 |
deferredPrompt = null;
|
| 1160 |
}
|
| 1161 |
|
| 1162 |
-
// iOS install hint
|
| 1163 |
function checkiOSInstall() {
|
| 1164 |
-
const isIOS = /iPad|iPhone|iPod/.test(navigator.userAgent)
|
|
|
|
| 1165 |
const isStandalone = window.navigator.standalone;
|
| 1166 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1167 |
const hint = document.createElement('div');
|
|
|
|
| 1168 |
hint.style.cssText = `
|
| 1169 |
position: fixed; bottom: 80px; left: 50%; transform: translateX(-50%);
|
| 1170 |
background: var(--surface2); border: 1px solid var(--accent);
|
| 1171 |
-
border-radius: 16px; padding:
|
| 1172 |
box-shadow: 0 0 30px var(--accent-glow); animation: fadeIn 0.5s ease;
|
| 1173 |
-
max-width: 90vw; text-align: center; font-size:
|
| 1174 |
`;
|
| 1175 |
hint.innerHTML = `
|
| 1176 |
-
<div style="margin-bottom:
|
| 1177 |
-
Tap <
|
| 1178 |
-
<
|
| 1179 |
-
<
|
| 1180 |
-
|
| 1181 |
-
|
| 1182 |
-
|
| 1183 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1184 |
`;
|
| 1185 |
document.body.appendChild(hint);
|
| 1186 |
}
|
|
|
|
| 2 |
<html lang="en">
|
| 3 |
<head>
|
| 4 |
<meta charset="UTF-8">
|
| 5 |
+
<meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=no, viewport-fit=cover">
|
| 6 |
<meta name="apple-mobile-web-app-capable" content="yes">
|
| 7 |
<meta name="apple-mobile-web-app-status-bar-style" content="black-translucent">
|
| 8 |
<meta name="apple-mobile-web-app-title" content="JARVIS">
|
| 9 |
<meta name="mobile-web-app-capable" content="yes">
|
| 10 |
<meta name="theme-color" content="#0a0e17">
|
| 11 |
+
<meta name="format-detection" content="telephone=no">
|
| 12 |
<link rel="manifest" href="/static/manifest.json">
|
| 13 |
+
<!-- Apple touch icons for iPhone / iPad -->
|
| 14 |
+
<link rel="apple-touch-icon" href="/static/apple-touch-icon.png">
|
| 15 |
+
<link rel="apple-touch-icon" sizes="152x152" href="/static/apple-touch-icon-152.png">
|
| 16 |
+
<link rel="apple-touch-icon" sizes="167x167" href="/static/apple-touch-icon-167.png">
|
| 17 |
+
<link rel="apple-touch-icon" sizes="180x180" href="/static/apple-touch-icon.png">
|
| 18 |
+
<!-- iOS splash screens -->
|
| 19 |
+
<meta name="apple-mobile-web-app-orientations" content="portrait">
|
| 20 |
<title>J.A.R.V.I.S.</title>
|
| 21 |
<style>
|
| 22 |
* { margin: 0; padding: 0; box-sizing: border-box; }
|
|
|
|
| 489 |
.welcome h2 { font-size: 20px; }
|
| 490 |
.quick-actions { gap: 6px; }
|
| 491 |
}
|
| 492 |
+
|
| 493 |
+
/* iOS safe areas — notch / Dynamic Island / home indicator */
|
| 494 |
+
@supports (padding-top: env(safe-area-inset-top)) {
|
| 495 |
+
.header {
|
| 496 |
+
padding-top: calc(12px + env(safe-area-inset-top));
|
| 497 |
+
}
|
| 498 |
+
.input-area {
|
| 499 |
+
padding-bottom: calc(16px + env(safe-area-inset-bottom));
|
| 500 |
+
}
|
| 501 |
+
}
|
| 502 |
+
|
| 503 |
+
/* iPad specific — wider layout */
|
| 504 |
+
@media (min-width: 768px) and (max-width: 1024px) {
|
| 505 |
+
.message { max-width: 75%; }
|
| 506 |
+
.input-row { max-width: 700px; }
|
| 507 |
+
}
|
| 508 |
</style>
|
| 509 |
</head>
|
| 510 |
<body>
|
|
|
|
| 1182 |
deferredPrompt = null;
|
| 1183 |
}
|
| 1184 |
|
| 1185 |
+
// iOS / iPadOS install hint
|
| 1186 |
function checkiOSInstall() {
|
| 1187 |
+
const isIOS = /iPad|iPhone|iPod/.test(navigator.userAgent) ||
|
| 1188 |
+
(navigator.platform === 'MacIntel' && navigator.maxTouchPoints > 1); // iPad with desktop UA
|
| 1189 |
const isStandalone = window.navigator.standalone;
|
| 1190 |
+
const dismissed = localStorage.getItem('jarvis_ios_install_dismissed');
|
| 1191 |
+
if (isIOS && !isStandalone && !dismissed) {
|
| 1192 |
+
const isIPad = /iPad/.test(navigator.userAgent) ||
|
| 1193 |
+
(navigator.platform === 'MacIntel' && navigator.maxTouchPoints > 1);
|
| 1194 |
+
const deviceName = isIPad ? 'iPad' : 'iPhone';
|
| 1195 |
const hint = document.createElement('div');
|
| 1196 |
+
hint.id = 'ios-install-hint';
|
| 1197 |
hint.style.cssText = `
|
| 1198 |
position: fixed; bottom: 80px; left: 50%; transform: translateX(-50%);
|
| 1199 |
background: var(--surface2); border: 1px solid var(--accent);
|
| 1200 |
+
border-radius: 16px; padding: 16px 24px; z-index: 1000;
|
| 1201 |
box-shadow: 0 0 30px var(--accent-glow); animation: fadeIn 0.5s ease;
|
| 1202 |
+
max-width: 90vw; text-align: center; font-size: 13px; color: var(--text);
|
| 1203 |
`;
|
| 1204 |
hint.innerHTML = `
|
| 1205 |
+
<div style="margin-bottom:8px; font-weight:bold; color: var(--accent); font-size:15px;">Install JARVIS on ${deviceName}</div>
|
| 1206 |
+
<div style="margin-bottom:6px;">Tap <svg width="16" height="16" viewBox="0 0 24 24" fill="currentColor" style="vertical-align:middle;"><path d="M16 5l-1.42 1.42-1.59-1.59V16h-1.98V4.83L9.42 6.42 8 5l4-4 4 4zm4 5v11c0 1.1-.9 2-2 2H6c-1.11 0-2-.9-2-2V10c0-1.11.89-2 2-2h3v2H6v11h12V10h-3V8h3c1.1 0 2 .89 2 2z"/></svg> Share → <strong>Add to Home Screen</strong></div>
|
| 1207 |
+
<div style="color: var(--text2); font-size:11px; margin-bottom:10px;">JARVIS will run like a native app — fullscreen, with mic & voice</div>
|
| 1208 |
+
<div style="display:flex; gap:8px; justify-content:center;">
|
| 1209 |
+
<button onclick="this.closest('#ios-install-hint').remove()" style="
|
| 1210 |
+
background:none; border:1px solid var(--accent);
|
| 1211 |
+
color:var(--accent); padding:6px 16px; border-radius:10px;
|
| 1212 |
+
font-family:inherit; font-size:12px; cursor:pointer;
|
| 1213 |
+
">GOT IT</button>
|
| 1214 |
+
<button onclick="localStorage.setItem('jarvis_ios_install_dismissed','1'); this.closest('#ios-install-hint').remove()" style="
|
| 1215 |
+
background:none; border:1px solid var(--text2);
|
| 1216 |
+
color:var(--text2); padding:6px 16px; border-radius:10px;
|
| 1217 |
+
font-family:inherit; font-size:11px; cursor:pointer;
|
| 1218 |
+
">DON'T SHOW AGAIN</button>
|
| 1219 |
+
</div>
|
| 1220 |
`;
|
| 1221 |
document.body.appendChild(hint);
|
| 1222 |
}
|
|
@@ -1,13 +1,32 @@
|
|
| 1 |
{
|
| 2 |
"name": "J.A.R.V.I.S.",
|
| 3 |
"short_name": "JARVIS",
|
| 4 |
-
"description": "Just A Rather Very Intelligent System",
|
| 5 |
"start_url": "/",
|
| 6 |
"display": "standalone",
|
| 7 |
"background_color": "#0a0e17",
|
| 8 |
"theme_color": "#00b4d8",
|
| 9 |
"orientation": "portrait",
|
|
|
|
| 10 |
"icons": [
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 11 |
{
|
| 12 |
"src": "/static/icon-192.png",
|
| 13 |
"sizes": "192x192",
|
|
|
|
| 1 |
{
|
| 2 |
"name": "J.A.R.V.I.S.",
|
| 3 |
"short_name": "JARVIS",
|
| 4 |
+
"description": "Just A Rather Very Intelligent System — Your AI Assistant",
|
| 5 |
"start_url": "/",
|
| 6 |
"display": "standalone",
|
| 7 |
"background_color": "#0a0e17",
|
| 8 |
"theme_color": "#00b4d8",
|
| 9 |
"orientation": "portrait",
|
| 10 |
+
"scope": "/",
|
| 11 |
"icons": [
|
| 12 |
+
{
|
| 13 |
+
"src": "/static/apple-touch-icon-152.png",
|
| 14 |
+
"sizes": "152x152",
|
| 15 |
+
"type": "image/png",
|
| 16 |
+
"purpose": "any"
|
| 17 |
+
},
|
| 18 |
+
{
|
| 19 |
+
"src": "/static/apple-touch-icon-167.png",
|
| 20 |
+
"sizes": "167x167",
|
| 21 |
+
"type": "image/png",
|
| 22 |
+
"purpose": "any"
|
| 23 |
+
},
|
| 24 |
+
{
|
| 25 |
+
"src": "/static/apple-touch-icon.png",
|
| 26 |
+
"sizes": "180x180",
|
| 27 |
+
"type": "image/png",
|
| 28 |
+
"purpose": "any"
|
| 29 |
+
},
|
| 30 |
{
|
| 31 |
"src": "/static/icon-192.png",
|
| 32 |
"sizes": "192x192",
|
|
@@ -1,6 +1,14 @@
|
|
| 1 |
// JARVIS Service Worker — enables offline + PWA install
|
| 2 |
-
const CACHE_NAME = 'jarvis-
|
| 3 |
-
const ASSETS = [
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4 |
|
| 5 |
self.addEventListener('install', (event) => {
|
| 6 |
event.waitUntil(
|
|
|
|
| 1 |
// JARVIS Service Worker — enables offline + PWA install
|
| 2 |
+
const CACHE_NAME = 'jarvis-v2';
|
| 3 |
+
const ASSETS = [
|
| 4 |
+
'/',
|
| 5 |
+
'/static/manifest.json',
|
| 6 |
+
'/static/icon-192.png',
|
| 7 |
+
'/static/icon-512.png',
|
| 8 |
+
'/static/apple-touch-icon.png',
|
| 9 |
+
'/static/apple-touch-icon-152.png',
|
| 10 |
+
'/static/apple-touch-icon-167.png',
|
| 11 |
+
];
|
| 12 |
|
| 13 |
self.addEventListener('install', (event) => {
|
| 14 |
event.waitUntil(
|
|
@@ -161,20 +161,28 @@ class TestRemindersTools(unittest.TestCase):
|
|
| 161 |
|
| 162 |
class TestCalendarTools(unittest.TestCase):
|
| 163 |
|
|
|
|
| 164 |
@patch("tools.app_automation._osascript")
|
| 165 |
-
def test_calendar_today(self, mock_osascript):
|
| 166 |
mock_osascript.return_value = "• 10:00 AM — Team Standup\n• 2:00 PM — Design Review"
|
| 167 |
from tools.app_automation import calendar_today
|
| 168 |
result = calendar_today()
|
| 169 |
self.assertIn("Standup", result)
|
| 170 |
|
|
|
|
| 171 |
@patch("tools.app_automation._osascript")
|
| 172 |
-
def test_calendar_create_event(self, mock_osascript):
|
| 173 |
mock_osascript.return_value = "Event created: Lunch"
|
| 174 |
from tools.app_automation import calendar_create_event
|
| 175 |
result = calendar_create_event("Lunch", "2026-04-07 12:00", "2026-04-07 13:00", location="Cafe")
|
| 176 |
self.assertIn("Event created", result)
|
| 177 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 178 |
|
| 179 |
class TestBrowserTools(unittest.TestCase):
|
| 180 |
|
|
@@ -289,6 +297,46 @@ class TestUniversalAppControl(unittest.TestCase):
|
|
| 289 |
self.assertIn("Times Square", result)
|
| 290 |
|
| 291 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 292 |
class TestTotalToolCount(unittest.TestCase):
|
| 293 |
"""Ensure total tools meets target after all modules loaded."""
|
| 294 |
|
|
|
|
| 161 |
|
| 162 |
class TestCalendarTools(unittest.TestCase):
|
| 163 |
|
| 164 |
+
@patch("tools.app_automation._check_calendar_permission", return_value=True)
|
| 165 |
@patch("tools.app_automation._osascript")
|
| 166 |
+
def test_calendar_today(self, mock_osascript, mock_perm):
|
| 167 |
mock_osascript.return_value = "• 10:00 AM — Team Standup\n• 2:00 PM — Design Review"
|
| 168 |
from tools.app_automation import calendar_today
|
| 169 |
result = calendar_today()
|
| 170 |
self.assertIn("Standup", result)
|
| 171 |
|
| 172 |
+
@patch("tools.app_automation._check_calendar_permission", return_value=True)
|
| 173 |
@patch("tools.app_automation._osascript")
|
| 174 |
+
def test_calendar_create_event(self, mock_osascript, mock_perm):
|
| 175 |
mock_osascript.return_value = "Event created: Lunch"
|
| 176 |
from tools.app_automation import calendar_create_event
|
| 177 |
result = calendar_create_event("Lunch", "2026-04-07 12:00", "2026-04-07 13:00", location="Cafe")
|
| 178 |
self.assertIn("Event created", result)
|
| 179 |
|
| 180 |
+
@patch("tools.app_automation._check_calendar_permission", return_value=False)
|
| 181 |
+
def test_calendar_permission_denied(self, mock_perm):
|
| 182 |
+
from tools.app_automation import calendar_today
|
| 183 |
+
result = calendar_today()
|
| 184 |
+
self.assertIn("Calendar access not granted", result)
|
| 185 |
+
|
| 186 |
|
| 187 |
class TestBrowserTools(unittest.TestCase):
|
| 188 |
|
|
|
|
| 297 |
self.assertIn("Times Square", result)
|
| 298 |
|
| 299 |
|
| 300 |
+
class TestOpenAppAliases(unittest.TestCase):
|
| 301 |
+
"""Test open_app resolves common aliases to correct macOS app names."""
|
| 302 |
+
|
| 303 |
+
@patch("tools.builtin.subprocess.run")
|
| 304 |
+
def test_camera_alias(self, mock_run):
|
| 305 |
+
mock_run.return_value = MagicMock(returncode=0, stdout="", stderr="")
|
| 306 |
+
from tools.builtin import open_app
|
| 307 |
+
result = open_app("camera")
|
| 308 |
+
self.assertIn("FaceTime", result)
|
| 309 |
+
mock_run.assert_called_once_with(
|
| 310 |
+
["open", "-a", "FaceTime"],
|
| 311 |
+
capture_output=True, text=True, timeout=10,
|
| 312 |
+
)
|
| 313 |
+
|
| 314 |
+
@patch("tools.builtin.subprocess.run")
|
| 315 |
+
def test_vscode_alias(self, mock_run):
|
| 316 |
+
mock_run.return_value = MagicMock(returncode=0, stdout="", stderr="")
|
| 317 |
+
from tools.builtin import open_app
|
| 318 |
+
result = open_app("vscode")
|
| 319 |
+
self.assertIn("Visual Studio Code", result)
|
| 320 |
+
|
| 321 |
+
@patch("tools.builtin.subprocess.run")
|
| 322 |
+
def test_direct_name_passthrough(self, mock_run):
|
| 323 |
+
mock_run.return_value = MagicMock(returncode=0, stdout="", stderr="")
|
| 324 |
+
from tools.builtin import open_app
|
| 325 |
+
result = open_app("Safari")
|
| 326 |
+
self.assertIn("Safari", result)
|
| 327 |
+
|
| 328 |
+
@patch("tools.builtin.subprocess.run")
|
| 329 |
+
def test_permission_error_message(self, mock_run):
|
| 330 |
+
mock_run.return_value = MagicMock(
|
| 331 |
+
returncode=1,
|
| 332 |
+
stdout="",
|
| 333 |
+
stderr="Not allowed to send Apple events",
|
| 334 |
+
)
|
| 335 |
+
from tools.builtin import open_app
|
| 336 |
+
result = open_app("Notes")
|
| 337 |
+
self.assertIn("Automation permission", result)
|
| 338 |
+
|
| 339 |
+
|
| 340 |
class TestTotalToolCount(unittest.TestCase):
|
| 341 |
"""Ensure total tools meets target after all modules loaded."""
|
| 342 |
|
|
@@ -1,13 +1,25 @@
|
|
| 1 |
"""JARVIS Tool System — extensible tool registry."""
|
| 2 |
|
| 3 |
import json
|
|
|
|
| 4 |
from typing import Callable
|
| 5 |
|
| 6 |
TOOL_REGISTRY: dict[str, dict] = {}
|
| 7 |
|
|
|
|
|
|
|
|
|
|
| 8 |
|
| 9 |
-
|
| 10 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 11 |
def decorator(func: Callable):
|
| 12 |
TOOL_REGISTRY[name] = {
|
| 13 |
"name": name,
|
|
@@ -15,13 +27,69 @@ def tool(name: str, description: str, parameters: dict = None):
|
|
| 15 |
"parameters": parameters or {},
|
| 16 |
"function": func,
|
| 17 |
}
|
|
|
|
|
|
|
| 18 |
return func
|
| 19 |
return decorator
|
| 20 |
|
| 21 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 22 |
async def execute_tool(name: str, args: dict) -> str:
|
| 23 |
if name not in TOOL_REGISTRY:
|
| 24 |
return f"Error: Unknown tool '{name}'"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 25 |
try:
|
| 26 |
result = TOOL_REGISTRY[name]["function"](**args)
|
| 27 |
if hasattr(result, "__await__"):
|
|
@@ -33,6 +101,48 @@ async def execute_tool(name: str, args: dict) -> str:
|
|
| 33 |
return f"Error executing {name}: {type(e).__name__}"
|
| 34 |
|
| 35 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 36 |
def get_tool_definitions() -> list[dict]:
|
| 37 |
"""Get tool definitions formatted for LLM function calling."""
|
| 38 |
tools = []
|
|
|
|
| 1 |
"""JARVIS Tool System — extensible tool registry."""
|
| 2 |
|
| 3 |
import json
|
| 4 |
+
import platform
|
| 5 |
from typing import Callable
|
| 6 |
|
| 7 |
TOOL_REGISTRY: dict[str, dict] = {}
|
| 8 |
|
| 9 |
+
# Tools that require macOS (osascript/subprocess) and should be delegated
|
| 10 |
+
# to a connected device when the server runs on Linux (HF Space / Render).
|
| 11 |
+
_MACOS_ONLY_TOOLS: set[str] = set()
|
| 12 |
|
| 13 |
+
IS_MACOS = platform.system() == "Darwin"
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
def tool(name: str, description: str, parameters: dict = None, macos_only: bool = False):
|
| 17 |
+
"""Decorator to register a tool.
|
| 18 |
+
|
| 19 |
+
Args:
|
| 20 |
+
macos_only: If True, this tool requires macOS (osascript, AppleScript, etc.)
|
| 21 |
+
and will be delegated to a connected device when running on Linux.
|
| 22 |
+
"""
|
| 23 |
def decorator(func: Callable):
|
| 24 |
TOOL_REGISTRY[name] = {
|
| 25 |
"name": name,
|
|
|
|
| 27 |
"parameters": parameters or {},
|
| 28 |
"function": func,
|
| 29 |
}
|
| 30 |
+
if macos_only:
|
| 31 |
+
_MACOS_ONLY_TOOLS.add(name)
|
| 32 |
return func
|
| 33 |
return decorator
|
| 34 |
|
| 35 |
|
| 36 |
+
async def _delegate_to_device(name: str, args: dict) -> str:
|
| 37 |
+
"""Delegate a macOS-only tool call to a connected device via WebSocket push."""
|
| 38 |
+
try:
|
| 39 |
+
from user_device_registry import list_devices, send_command_to_device
|
| 40 |
+
except ImportError:
|
| 41 |
+
return f"Error: {name} requires macOS but this server runs on Linux. No device delegation available."
|
| 42 |
+
|
| 43 |
+
# Find an online device to delegate to
|
| 44 |
+
devices = await list_devices("default")
|
| 45 |
+
online_devices = [d for d in devices if d.get("status") == "online"]
|
| 46 |
+
if not online_devices:
|
| 47 |
+
return (
|
| 48 |
+
f"Error: '{name}' requires macOS but this server runs in the cloud. "
|
| 49 |
+
f"No connected Mac device found to execute this command. "
|
| 50 |
+
f"Make sure the JARVIS listener is running on your Mac."
|
| 51 |
+
)
|
| 52 |
+
|
| 53 |
+
target = online_devices[0]
|
| 54 |
+
device_id = target.get("device_id", "")
|
| 55 |
+
alias = target.get("alias", "device")
|
| 56 |
+
|
| 57 |
+
# Try WebSocket push first (instant)
|
| 58 |
+
try:
|
| 59 |
+
# Import at call time to avoid circular imports
|
| 60 |
+
import importlib
|
| 61 |
+
server_mod = importlib.import_module("server")
|
| 62 |
+
push_fn = getattr(server_mod, "push_command_to_device", None)
|
| 63 |
+
if push_fn:
|
| 64 |
+
pushed = await push_fn(device_id, {
|
| 65 |
+
"action": "execute_tool",
|
| 66 |
+
"tool": name,
|
| 67 |
+
"args": args,
|
| 68 |
+
})
|
| 69 |
+
if pushed:
|
| 70 |
+
return f"Command '{name}' sent to {alias} for execution."
|
| 71 |
+
except Exception:
|
| 72 |
+
pass
|
| 73 |
+
|
| 74 |
+
# Fallback: queue command via REST
|
| 75 |
+
result = await send_command_to_device(
|
| 76 |
+
target_alias=alias,
|
| 77 |
+
command=json.dumps({"tool": name, "args": args}),
|
| 78 |
+
user_id="default",
|
| 79 |
+
)
|
| 80 |
+
if "error" in result:
|
| 81 |
+
return f"Error delegating '{name}' to {alias}: {result['error']}"
|
| 82 |
+
return f"Command '{name}' queued for execution on {alias}."
|
| 83 |
+
|
| 84 |
+
|
| 85 |
async def execute_tool(name: str, args: dict) -> str:
|
| 86 |
if name not in TOOL_REGISTRY:
|
| 87 |
return f"Error: Unknown tool '{name}'"
|
| 88 |
+
|
| 89 |
+
# If running on Linux (HF Space) and this tool needs macOS, delegate it
|
| 90 |
+
if not IS_MACOS and name in _MACOS_ONLY_TOOLS:
|
| 91 |
+
return await _delegate_to_device(name, args)
|
| 92 |
+
|
| 93 |
try:
|
| 94 |
result = TOOL_REGISTRY[name]["function"](**args)
|
| 95 |
if hasattr(result, "__await__"):
|
|
|
|
| 101 |
return f"Error executing {name}: {type(e).__name__}"
|
| 102 |
|
| 103 |
|
| 104 |
+
def register_macos_tools():
|
| 105 |
+
"""Register all macOS-only tool names.
|
| 106 |
+
|
| 107 |
+
Called after all tool modules are imported so TOOL_REGISTRY is populated.
|
| 108 |
+
These tools use osascript, AppleScript, or macOS-specific CLI commands.
|
| 109 |
+
On Linux (HF Space / Render), they get automatically delegated to a
|
| 110 |
+
connected Mac device via WebSocket push.
|
| 111 |
+
"""
|
| 112 |
+
macos_tools = {
|
| 113 |
+
# system_control.py — all 22 tools
|
| 114 |
+
"set_volume", "get_volume", "set_brightness", "toggle_dark_mode",
|
| 115 |
+
"wifi_control", "bluetooth_control", "do_not_disturb", "screenshot",
|
| 116 |
+
"media_control", "clipboard", "send_notification", "list_running_apps",
|
| 117 |
+
"quit_app", "set_timer", "lock_screen", "sleep_display", "empty_trash",
|
| 118 |
+
"announce", "open_url", "search_files", "create_reminder", "get_events",
|
| 119 |
+
# app_automation.py — all 36 tools
|
| 120 |
+
"spotify_play", "spotify_play_uri", "spotify_search", "spotify_queue",
|
| 121 |
+
"spotify_status", "spotify_control",
|
| 122 |
+
"notes_create", "notes_append", "notes_list", "notes_search",
|
| 123 |
+
"notes_read", "notes_delete",
|
| 124 |
+
"reminders_add", "reminders_list", "reminders_complete",
|
| 125 |
+
"calendar_create_event", "calendar_today",
|
| 126 |
+
"send_imessage", "read_messages",
|
| 127 |
+
"mail_compose", "mail_unread",
|
| 128 |
+
"browser_open", "browser_tabs", "browser_read_page",
|
| 129 |
+
"spotlight_search", "finder_open", "finder_move", "finder_copy", "trash_file",
|
| 130 |
+
"contacts_search", "maps_directions",
|
| 131 |
+
"app_keystroke", "app_menu_click", "app_window_manage", "textedit_create",
|
| 132 |
+
# builtin.py — macOS-specific subset
|
| 133 |
+
"open_app", "open_terminal",
|
| 134 |
+
# vscode_tools.py — all 7 tools
|
| 135 |
+
"vscode_open", "vscode_open_terminal", "vscode_run_command",
|
| 136 |
+
"copilot_chat", "copilot_inline", "vscode_list_extensions", "vscode_diff",
|
| 137 |
+
# device_control.py — macOS-specific subset
|
| 138 |
+
"bluetooth_scan", "bluetooth_pair", "bluetooth_connect",
|
| 139 |
+
"bluetooth_disconnect", "bluetooth_info", "network_scan",
|
| 140 |
+
"bonjour_discover", "homekit_control", "homekit_list_shortcuts",
|
| 141 |
+
"find_my_devices", "siri_command", "airdrop_file",
|
| 142 |
+
}
|
| 143 |
+
_MACOS_ONLY_TOOLS.update(macos_tools)
|
| 144 |
+
|
| 145 |
+
|
| 146 |
def get_tool_definitions() -> list[dict]:
|
| 147 |
"""Get tool definitions formatted for LLM function calling."""
|
| 148 |
tools = []
|
|
@@ -44,6 +44,10 @@ def _osascript(script: str, timeout: int = 15) -> str:
|
|
| 44 |
out = result.stdout.strip()
|
| 45 |
if result.returncode != 0 and result.stderr.strip():
|
| 46 |
err = result.stderr.strip()
|
|
|
|
|
|
|
|
|
|
|
|
|
| 47 |
if "execution error" in err.lower():
|
| 48 |
return f"Error: {err}"
|
| 49 |
return out
|
|
@@ -605,6 +609,26 @@ def reminders_complete(title: str) -> str:
|
|
| 605 |
# APPLE CALENDAR
|
| 606 |
# ═══════════════════════════════════════════════════════════════
|
| 607 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 608 |
|
| 609 |
@tool(
|
| 610 |
name="calendar_create_event",
|
|
@@ -625,6 +649,9 @@ def reminders_complete(title: str) -> str:
|
|
| 625 |
def calendar_create_event(title: str, start_date: str, end_date: str,
|
| 626 |
location: str = "", notes: str = "",
|
| 627 |
calendar_name: str = "") -> str:
|
|
|
|
|
|
|
|
|
|
| 628 |
safe_title = title.replace('"', '\\"')
|
| 629 |
safe_loc = location.replace('"', '\\"')
|
| 630 |
safe_notes = notes.replace('"', '\\"')
|
|
@@ -654,6 +681,9 @@ def calendar_create_event(title: str, start_date: str, end_date: str,
|
|
| 654 |
parameters={"type": "object", "properties": {}},
|
| 655 |
)
|
| 656 |
def calendar_today() -> str:
|
|
|
|
|
|
|
|
|
|
| 657 |
result = _osascript('''
|
| 658 |
set todayStart to current date
|
| 659 |
set time of todayStart to 0
|
|
|
|
| 44 |
out = result.stdout.strip()
|
| 45 |
if result.returncode != 0 and result.stderr.strip():
|
| 46 |
err = result.stderr.strip()
|
| 47 |
+
if "not allowed assistive access" in err.lower() or "not allowed to send keystrokes" in err.lower():
|
| 48 |
+
return ("Error: JARVIS needs Automation/Accessibility permission. "
|
| 49 |
+
"Go to System Settings → Privacy & Security → Automation and enable JARVIS, "
|
| 50 |
+
"then also check Accessibility.")
|
| 51 |
if "execution error" in err.lower():
|
| 52 |
return f"Error: {err}"
|
| 53 |
return out
|
|
|
|
| 609 |
# APPLE CALENDAR
|
| 610 |
# ═══════════════════════════════════════════════════════════════
|
| 611 |
|
| 612 |
+
# Cache so Calendar permission is only checked once per session
|
| 613 |
+
_calendar_permission_checked = False
|
| 614 |
+
_calendar_permission_granted = False
|
| 615 |
+
|
| 616 |
+
|
| 617 |
+
def _check_calendar_permission() -> bool:
|
| 618 |
+
"""Check Calendar automation permission once; cache the result."""
|
| 619 |
+
global _calendar_permission_checked, _calendar_permission_granted
|
| 620 |
+
if _calendar_permission_checked:
|
| 621 |
+
return _calendar_permission_granted
|
| 622 |
+
_calendar_permission_checked = True
|
| 623 |
+
result = subprocess.run(
|
| 624 |
+
["osascript", "-e", 'tell application "Calendar" to name'],
|
| 625 |
+
capture_output=True, text=True, timeout=10,
|
| 626 |
+
)
|
| 627 |
+
_calendar_permission_granted = result.returncode == 0
|
| 628 |
+
if not _calendar_permission_granted:
|
| 629 |
+
_calendar_permission_granted = False
|
| 630 |
+
return _calendar_permission_granted
|
| 631 |
+
|
| 632 |
|
| 633 |
@tool(
|
| 634 |
name="calendar_create_event",
|
|
|
|
| 649 |
def calendar_create_event(title: str, start_date: str, end_date: str,
|
| 650 |
location: str = "", notes: str = "",
|
| 651 |
calendar_name: str = "") -> str:
|
| 652 |
+
if not _check_calendar_permission():
|
| 653 |
+
return ("Error: Calendar access not granted. Go to System Settings → Privacy & Security → "
|
| 654 |
+
"Automation and allow JARVIS to control Calendar.")
|
| 655 |
safe_title = title.replace('"', '\\"')
|
| 656 |
safe_loc = location.replace('"', '\\"')
|
| 657 |
safe_notes = notes.replace('"', '\\"')
|
|
|
|
| 681 |
parameters={"type": "object", "properties": {}},
|
| 682 |
)
|
| 683 |
def calendar_today() -> str:
|
| 684 |
+
if not _check_calendar_permission():
|
| 685 |
+
return ("Error: Calendar access not granted. Go to System Settings → Privacy & Security → "
|
| 686 |
+
"Automation and allow JARVIS to control Calendar.")
|
| 687 |
result = _osascript('''
|
| 688 |
set todayStart to current date
|
| 689 |
set time of todayStart to 0
|
|
@@ -367,21 +367,52 @@ def open_app(app_name: str) -> str:
|
|
| 367 |
system = platform.system()
|
| 368 |
try:
|
| 369 |
if system == "Darwin":
|
| 370 |
-
#
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 371 |
result = subprocess.run(
|
| 372 |
-
["open", "-a",
|
| 373 |
capture_output=True, text=True, timeout=10,
|
| 374 |
)
|
| 375 |
if result.returncode == 0:
|
| 376 |
-
return f"Opened {
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 377 |
# Fallback: AppleScript activate (handles apps `open -a` can't find)
|
| 378 |
fallback = subprocess.run(
|
| 379 |
-
["osascript", "-e", f'tell application "{
|
| 380 |
capture_output=True, text=True, timeout=10,
|
| 381 |
)
|
| 382 |
if fallback.returncode == 0:
|
| 383 |
-
return f"Opened {
|
| 384 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 385 |
elif system == "Linux":
|
| 386 |
subprocess.Popen([app_name])
|
| 387 |
elif system == "Windows":
|
|
|
|
| 367 |
system = platform.system()
|
| 368 |
try:
|
| 369 |
if system == "Darwin":
|
| 370 |
+
# Map common aliases to actual macOS app names
|
| 371 |
+
app_aliases = {
|
| 372 |
+
"camera": "FaceTime",
|
| 373 |
+
"photo booth": "Photo Booth",
|
| 374 |
+
"facetime": "FaceTime",
|
| 375 |
+
"vscode": "Visual Studio Code",
|
| 376 |
+
"vs code": "Visual Studio Code",
|
| 377 |
+
"chrome": "Google Chrome",
|
| 378 |
+
"word": "Microsoft Word",
|
| 379 |
+
"excel": "Microsoft Excel",
|
| 380 |
+
"powerpoint": "Microsoft PowerPoint",
|
| 381 |
+
"teams": "Microsoft Teams",
|
| 382 |
+
"outlook": "Microsoft Outlook",
|
| 383 |
+
"code": "Visual Studio Code",
|
| 384 |
+
"iterm": "iTerm",
|
| 385 |
+
"terminal": "Terminal",
|
| 386 |
+
}
|
| 387 |
+
resolved_name = app_aliases.get(app_name.lower(), app_name)
|
| 388 |
+
|
| 389 |
+
# Primary: use macOS `open -a` (no Automation permission needed)
|
| 390 |
result = subprocess.run(
|
| 391 |
+
["open", "-a", resolved_name],
|
| 392 |
capture_output=True, text=True, timeout=10,
|
| 393 |
)
|
| 394 |
if result.returncode == 0:
|
| 395 |
+
return f"Opened {resolved_name}"
|
| 396 |
+
# Retry with original name if alias didn't work
|
| 397 |
+
if resolved_name != app_name:
|
| 398 |
+
result2 = subprocess.run(
|
| 399 |
+
["open", "-a", app_name],
|
| 400 |
+
capture_output=True, text=True, timeout=10,
|
| 401 |
+
)
|
| 402 |
+
if result2.returncode == 0:
|
| 403 |
+
return f"Opened {app_name}"
|
| 404 |
# Fallback: AppleScript activate (handles apps `open -a` can't find)
|
| 405 |
fallback = subprocess.run(
|
| 406 |
+
["osascript", "-e", f'tell application "{resolved_name}" to activate'],
|
| 407 |
capture_output=True, text=True, timeout=10,
|
| 408 |
)
|
| 409 |
if fallback.returncode == 0:
|
| 410 |
+
return f"Opened {resolved_name}"
|
| 411 |
+
err_msg = result.stderr.strip() or fallback.stderr.strip()
|
| 412 |
+
if "not allowed" in err_msg.lower():
|
| 413 |
+
return (f"Error opening {app_name}: JARVIS needs Automation permission. "
|
| 414 |
+
"Go to System Settings → Privacy & Security → Automation and enable JARVIS.")
|
| 415 |
+
return f"Error opening {app_name}: {err_msg}"
|
| 416 |
elif system == "Linux":
|
| 417 |
subprocess.Popen([app_name])
|
| 418 |
elif system == "Windows":
|
|
@@ -23,7 +23,12 @@ sleep 1
|
|
| 23 |
# ─── Remove JARVIS .app bundles ───
|
| 24 |
rm -rf "$HOME/jarvis/JARVIS-Listener.app"
|
| 25 |
rm -rf "/Applications/JARVIS Listener.app"
|
| 26 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 27 |
|
| 28 |
# ─── Remove the Siri Shortcut ───
|
| 29 |
# Delete the generated shortcut file
|
|
@@ -68,16 +73,16 @@ rm -rf "$HOME/jarvis/tools/__pycache__"
|
|
| 68 |
tccutil reset Microphone ai.jarvis.listener 2>/dev/null || true
|
| 69 |
|
| 70 |
echo ""
|
| 71 |
-
echo " ╔═══════════════════════════════════════╗"
|
| 72 |
-
echo " ║ J.A.R.V.I.S. UNINSTALLED
|
| 73 |
-
echo " ║
|
| 74 |
-
echo " ║ ✓ Services stopped & removed
|
| 75 |
-
echo " ║ ✓ Mac apps removed
|
| 76 |
-
echo " ║ ✓ Siri Shortcut removed
|
| 77 |
-
echo " ║ ✓ Log files cleaned
|
| 78 |
-
echo " ║ ✓ Build artifacts removed
|
| 79 |
-
echo " ║
|
| 80 |
-
echo " ║ Source code preserved at ~/jarvis/
|
| 81 |
echo " ║ To reinstall: bash ~/jarvis/install.sh║"
|
| 82 |
-
echo " ╚═══════════════════════════════════════╝"
|
| 83 |
echo ""
|
|
|
|
| 23 |
# ─── Remove JARVIS .app bundles ───
|
| 24 |
rm -rf "$HOME/jarvis/JARVIS-Listener.app"
|
| 25 |
rm -rf "/Applications/JARVIS Listener.app"
|
| 26 |
+
if [ -d "/Applications/JARVIS.app" ]; then
|
| 27 |
+
sudo rm -rf "/Applications/JARVIS.app" 2>/dev/null || {
|
| 28 |
+
echo " [!] Could not remove /Applications/JARVIS.app (need admin password)"
|
| 29 |
+
echo " [!] Run: sudo rm -rf /Applications/JARVIS.app"
|
| 30 |
+
}
|
| 31 |
+
fi
|
| 32 |
|
| 33 |
# ─── Remove the Siri Shortcut ───
|
| 34 |
# Delete the generated shortcut file
|
|
|
|
| 73 |
tccutil reset Microphone ai.jarvis.listener 2>/dev/null || true
|
| 74 |
|
| 75 |
echo ""
|
| 76 |
+
echo " ╔═════════════════════════════════════════╗"
|
| 77 |
+
echo " ║ J.A.R.V.I.S. UNINSTALLED ║"
|
| 78 |
+
echo " ║ ║"
|
| 79 |
+
echo " ║ ✓ Services stopped & removed ║"
|
| 80 |
+
echo " ║ ✓ Mac apps removed ║"
|
| 81 |
+
echo " ║ ✓ Siri Shortcut removed ║"
|
| 82 |
+
echo " ║ ✓ Log files cleaned ║"
|
| 83 |
+
echo " ║ ✓ Build artifacts removed ║"
|
| 84 |
+
echo " ║ ║"
|
| 85 |
+
echo " ║ Source code preserved at ~/jarvis/ ║"
|
| 86 |
echo " ║ To reinstall: bash ~/jarvis/install.sh║"
|
| 87 |
+
echo " ╚═════════════════════════════════════════╝"
|
| 88 |
echo ""
|