Khanna, Videh Rakesh Rakesh commited on
Commit
2aa3ea6
·
1 Parent(s): 81ba037

feat: cross-device support — all Apple devices, cloud tool delegation, PWA

Browse files

- Fix uninstall permission errors (sudo for /Applications)
- Fix mac app not opening (hardcode JARVIS_DIR in C launcher)
- Fix mic staying on after 'bye' (standby mode, menu bar indicator)
- Fix calendar permission spam (cached permission checks)
- Fix app opening (alias mapping: camera→FaceTime, vscode→VS Code, etc.)
- Add startup automation permission requests from JARVIS process
- Server error retry prompt in conversation loop

Cloud tool delegation:
- Tag 78 macOS-only tools (osascript/subprocess)
- Auto-delegate to connected Mac via WebSocket when server runs on Linux
- Listener connects to server WebSocket, executes tools locally, returns results
- 26 tools run directly in cloud (web search, weather, tasks, memory, etc.)

iPhone/iPad/Apple Watch:
- Siri Shortcut now hits HF Space (not localhost) — works on all Apple devices
- Shortcut: dictate → POST /api/ask → speak response → notification
- PWA: apple-touch-icons (152/167/180px), safe area insets, iPad layout
- iOS install prompt with device detection, don't-show-again option
- manifest.json with full icon set, service worker v2

WebView app:
- Try local server → HF Space → Render → error page
- No local HTML dependency, loads server UI directly

.gitignore CHANGED
@@ -13,3 +13,4 @@ JARVIS-Listener.app/
13
  JARVIS.app/
14
  *.icns
15
  *.shortcut
 
 
13
  JARVIS.app/
14
  *.icns
15
  *.shortcut
16
+ .device_id
install.sh CHANGED
@@ -90,28 +90,26 @@ fi
90
 
91
  # Create the native launcher binary
92
  # A compiled binary means macOS TCC properly identifies this app for mic permission
93
- cat > /tmp/jarvis_launcher.c << 'CSOURCE'
 
94
  #include <unistd.h>
95
- #include <libgen.h>
96
- #include <string.h>
97
  #include <stdio.h>
98
  #include <stdlib.h>
99
- #include <mach-o/dyld.h>
 
100
 
101
  int main(int argc, char *argv[]) {
102
- char exe_path[4096];
103
- uint32_t size = sizeof(exe_path);
104
- _NSGetExecutablePath(exe_path, &size);
105
- char *dir = dirname(exe_path);
106
- dir = dirname(dir);
107
- dir = dirname(dir);
108
- dir = dirname(dir);
109
  char python_path[4096];
110
  char script_path[4096];
111
- snprintf(python_path, sizeof(python_path), "%s/venv/bin/python", dir);
112
- snprintf(script_path, sizeof(script_path), "%s/jarvis_listener.py", dir);
113
- chdir(dir);
114
- execl(python_path, python_path, script_path, NULL);
 
 
 
 
 
115
  perror("Failed to launch JARVIS listener");
116
  return 1;
117
  }
@@ -196,26 +194,20 @@ if [ -f "$JARVIS_DIR/static/JARVIS.icns" ]; then
196
  fi
197
 
198
  # Create launcher that runs jarvis_app.py (the WebView)
199
- cat > /tmp/jarvis_app_launcher.c << 'CSOURCE'
 
200
  #include <unistd.h>
201
  #include <stdio.h>
202
  #include <stdlib.h>
203
- #include <mach-o/dyld.h>
204
- #include <libgen.h>
205
 
206
  int main(int argc, char *argv[]) {
207
- char exe_path[4096];
208
- uint32_t size = sizeof(exe_path);
209
- _NSGetExecutablePath(exe_path, &size);
210
- char *dir = dirname(exe_path);
211
- dir = dirname(dir);
212
- dir = dirname(dir);
213
- dir = dirname(dir);
214
  char python_path[4096];
215
  char script_path[4096];
216
- snprintf(python_path, sizeof(python_path), "%s/venv/bin/python", dir);
217
- snprintf(script_path, sizeof(script_path), "%s/jarvis_app.py", dir);
218
- chdir(dir);
219
  execl(python_path, python_path, script_path, NULL);
220
  perror("Failed to launch JARVIS app");
221
  return 1;
@@ -341,10 +333,18 @@ except Exception as e:
341
 
342
  # 6e. AppleScript / Automation (Notes, Calendar, Mail, Finder, etc.)
343
  echo " Requesting automation access (Notes, Calendar, Mail, Finder)..."
344
- for APP_NAME in "Notes" "Calendar" "Reminders" "Mail" "Finder" "System Events"; do
 
 
 
 
 
 
 
 
345
  osascript -e "tell application \"$APP_NAME\" to name" 2>/dev/null && \
346
- echo " ✓ Automation: $APP_NAME" || \
347
- echo " [!] Automation: $APP_NAME — grant when prompted"
348
  done
349
 
350
  # 6f. Contacts
@@ -520,28 +520,28 @@ echo " ✓ All services launched (listener includes menu bar)"
520
 
521
  echo ""
522
  echo " ╔═══════════════════════════════════════╗"
523
- echo " ║ J.A.R.V.I.S. INSTALLED ║"
524
  echo " ║ ║"
525
- echo " ║ ✓ Server: auto-starts on boot ║"
526
- echo " ║ ✓ Listener: wake+clap+hotkey+menu ║"
527
- echo " ║ ✓ JARVIS.app: in /Applications ║"
528
- echo " ║ ✓ Menu bar: J icon (unified) ║"
529
- echo " ║ ✓ Siri Shortcut: Hey Siri, JARVIS ║"
530
- echo " ║ ✓ Clap/Snap: double-clap to wake ║"
531
- echo " ║ ✓ All permissions: requested ║"
532
  echo " ║ ║"
533
  echo " ║ Activation: ║"
534
- echo " ║ • Open JARVIS.app (WebView UI) ║"
535
- echo " ║ • Say 'Hey Siri, JARVIS' ║"
536
- echo " ║ • Say 'Jarvis' (local wake word) ║"
537
- echo " ║ • Ctrl+Shift+J (global hotkey) ║"
538
- echo " ║ • Double clap or snap ║"
539
- echo " ║ • Click J in menu bar ║"
540
  echo " ║ ║"
541
  echo " ║ Conversation mode: ║"
542
  echo " ║ • After wake, mic stays on ║"
543
  echo " ║ • Ask follow-up questions ║"
544
- echo " ║ • Say 'goodbye' to end session ║"
545
  echo " ║ ║"
546
  echo " ║ To uninstall: ║"
547
  echo " ║ bash ~/jarvis/uninstall.sh ║"
 
90
 
91
  # Create the native launcher binary
92
  # A compiled binary means macOS TCC properly identifies this app for mic permission
93
+ # Uses JARVIS_DIR hardcoded at build time so it works even if .app is moved
94
+ cat > /tmp/jarvis_launcher.c << CSOURCE
95
  #include <unistd.h>
 
 
96
  #include <stdio.h>
97
  #include <stdlib.h>
98
+
99
+ #define JARVIS_DIR "$JARVIS_DIR"
100
 
101
  int main(int argc, char *argv[]) {
 
 
 
 
 
 
 
102
  char python_path[4096];
103
  char script_path[4096];
104
+ snprintf(python_path, sizeof(python_path), "%s/venv/bin/python", JARVIS_DIR);
105
+ snprintf(script_path, sizeof(script_path), "%s/jarvis_listener.py", JARVIS_DIR);
106
+ chdir(JARVIS_DIR);
107
+ /* Forward any extra args (e.g. --no-greet) */
108
+ if (argc > 1) {
109
+ execl(python_path, python_path, script_path, argv[1], NULL);
110
+ } else {
111
+ execl(python_path, python_path, script_path, NULL);
112
+ }
113
  perror("Failed to launch JARVIS listener");
114
  return 1;
115
  }
 
194
  fi
195
 
196
  # Create launcher that runs jarvis_app.py (the WebView)
197
+ # Uses JARVIS_DIR hardcoded at build time so it works from /Applications too
198
+ cat > /tmp/jarvis_app_launcher.c << CSOURCE
199
  #include <unistd.h>
200
  #include <stdio.h>
201
  #include <stdlib.h>
202
+
203
+ #define JARVIS_DIR "$JARVIS_DIR"
204
 
205
  int main(int argc, char *argv[]) {
 
 
 
 
 
 
 
206
  char python_path[4096];
207
  char script_path[4096];
208
+ snprintf(python_path, sizeof(python_path), "%s/venv/bin/python", JARVIS_DIR);
209
+ snprintf(script_path, sizeof(script_path), "%s/jarvis_app.py", JARVIS_DIR);
210
+ chdir(JARVIS_DIR);
211
  execl(python_path, python_path, script_path, NULL);
212
  perror("Failed to launch JARVIS app");
213
  return 1;
 
333
 
334
  # 6e. AppleScript / Automation (Notes, Calendar, Mail, Finder, etc.)
335
  echo " Requesting automation access (Notes, Calendar, Mail, Finder)..."
336
+ echo " NOTE: Automation permissions are requested when JARVIS first starts."
337
+ echo " When JARVIS launches, macOS will prompt you to allow it to control"
338
+ echo " each app. Click 'OK' or 'Allow' for each prompt."
339
+ echo ""
340
+ echo " If you don't see prompts, go to:"
341
+ echo " System Settings → Privacy & Security → Automation → enable JARVIS"
342
+ echo ""
343
+ # Still trigger from Terminal for any that might work
344
+ for APP_NAME in "System Events" "Finder"; do
345
  osascript -e "tell application \"$APP_NAME\" to name" 2>/dev/null && \
346
+ echo " ✓ Automation: $APP_NAME (Terminal)" || \
347
+ echo " [i] Automation: $APP_NAME — will prompt when JARVIS starts"
348
  done
349
 
350
  # 6f. Contacts
 
520
 
521
  echo ""
522
  echo " ╔═══════════════════════════════════════╗"
523
+ echo " ║ J.A.R.V.I.S. INSTALLED ║"
524
  echo " ║ ║"
525
+ echo " ║ ✓ Server: auto-starts on boot ║"
526
+ echo " ║ ✓ Listener: wake+clap+hotkey+menu ║"
527
+ echo " ║ ✓ JARVIS.app: in /Applications ║"
528
+ echo " ║ ✓ Menu bar: J icon (unified) ║"
529
+ echo " ║ ✓ Siri Shortcut: Hey Siri, JARVIS ║"
530
+ echo " ║ ✓ Clap/Snap: double-clap to wake ║"
531
+ echo " ║ ✓ All permissions: requested ║"
532
  echo " ║ ║"
533
  echo " ║ Activation: ║"
534
+ echo " ║ • Open JARVIS.app (WebView UI) ║"
535
+ echo " ║ • Say 'Hey Siri, JARVIS' ║"
536
+ echo " ║ • Say 'Jarvis' (local wake word) ║"
537
+ echo " ║ • Ctrl+Shift+J (global hotkey) ║"
538
+ echo " ║ • Double clap or snap ║"
539
+ echo " ║ • Click J in menu bar ║"
540
  echo " ║ ║"
541
  echo " ║ Conversation mode: ║"
542
  echo " ║ • After wake, mic stays on ║"
543
  echo " ║ • Ask follow-up questions ║"
544
+ echo " ║ • Say 'goodbye' to end session ║"
545
  echo " ║ ║"
546
  echo " ║ To uninstall: ║"
547
  echo " ║ bash ~/jarvis/uninstall.sh ║"
install_siri_shortcut.py CHANGED
@@ -1,98 +1,225 @@
1
  #!/usr/bin/env python3
2
- """Generate and install the local macOS JARVIS Siri Shortcut.
3
 
4
- The shortcut first ensures the JARVIS listener is running (in case it
5
- auto-quit after idle), then triggers the microphone command capture.
 
 
6
 
7
- Setup: Say "Hey Siri, JARVIS" Apple Siri activatesShortcut fires
8
- listener starts (if needed) → mic ON → listen → respond → mic OFF.
 
 
 
9
  """
10
 
11
  import plistlib
12
  import subprocess
13
  import os
 
 
 
14
 
15
  JARVIS_DIR = os.path.dirname(os.path.abspath(__file__))
16
- TRIGGER_URL = "http://127.0.0.1:8111/trigger"
17
- VENV_PYTHON = os.path.join(JARVIS_DIR, "venv", "bin", "python")
18
- LISTENER_SCRIPT = os.path.join(JARVIS_DIR, "jarvis_listener.py")
19
-
20
-
21
- shortcut = {
22
- "WFWorkflowMinimumClientVersionString": "900",
23
- "WFWorkflowMinimumClientVersion": 900,
24
- "WFWorkflowClientVersion": "2302.0.4",
25
- "WFWorkflowHasShortcutInputVariables": False,
26
- "WFWorkflowImportQuestions": [],
27
- "WFWorkflowName": "JARVIS",
28
- "WFWorkflowIcon": {
29
- "WFWorkflowIconStartColor": 463140863, # Blue
30
- "WFWorkflowIconGlyphNumber": 59511, # Microphone icon
31
- },
32
- "WFWorkflowTypes": [
33
- "NCWidget", # Notification Center
34
- "WatchKit", # Apple Watch
35
- ],
36
- "WFWorkflowInputContentItemClasses": [
37
- "WFStringContentItem",
38
- ],
39
- "WFWorkflowActions": [
40
- # ── Action 1: Set the trigger URL ──
41
  {
42
- "WFWorkflowActionIdentifier": "is.workflow.actions.url",
43
  "WFWorkflowActionParameters": {
44
- "WFURLActionURL": TRIGGER_URL,
 
45
  },
46
  },
47
- # ── Action 2: GET the trigger URL to wake the listener ──
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
48
  {
49
  "WFWorkflowActionIdentifier": "is.workflow.actions.downloadurl",
50
  "WFWorkflowActionParameters": {
51
- "WFHTTPMethod": "GET",
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
52
  },
53
  },
54
- ],
55
- }
56
-
57
- # Write the unsigned .shortcut file (temporary)
58
- unsigned_path = os.path.join(JARVIS_DIR, "JARVIS-unsigned.shortcut")
59
- with open(unsigned_path, "wb") as f:
60
- plistlib.dump(shortcut, f, fmt=plistlib.FMT_BINARY)
61
-
62
- print(f"Trigger URL: {TRIGGER_URL}")
63
- print()
64
-
65
- # Sign the shortcut — required by modern macOS for import.
66
- # "people-who-know-me" mode works without developer signing identity.
67
- final_path = os.path.join(JARVIS_DIR, "JARVIS.shortcut")
68
- try:
69
- subprocess.run(
70
- ["shortcuts", "sign", "-m", "people-who-know-me", "-i", unsigned_path, "-o", final_path],
71
- check=True, capture_output=True,
72
- )
73
- os.remove(unsigned_path)
74
- print(f"Signed shortcut: {final_path}")
75
- except Exception as e:
76
- print(f"Signing failed: {e}")
77
- # Fall back to unsigned (may not import on modern macOS)
78
- os.rename(unsigned_path, final_path)
79
- print(f"Unsigned shortcut: {final_path}")
80
-
81
- # Auto-import: try to open the shortcut directly which triggers Shortcuts app import dialog
82
- print()
83
- print("Importing shortcut into Shortcuts app...")
84
- try:
85
- result = subprocess.run(
86
- ["open", final_path],
87
- capture_output=True, text=True, timeout=10,
88
- )
89
- if result.returncode == 0:
90
- print("Shortcuts app should open with an import prompt — click 'Add Shortcut'.")
91
- else:
92
- print(f"Auto-import failed: {result.stderr.strip()}")
93
- print(f"Manual import: open -a Shortcuts \"{final_path}\"")
94
- except Exception as e:
95
- print(f"Auto-import error: {e}")
96
-
97
- print()
98
- print('After import, say "Hey Siri, JARVIS" to wake the local listener.')
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
  #!/usr/bin/env python3
2
+ """Generate and install JARVIS Siri Shortcuts for ALL Apple devices.
3
 
4
+ Creates two shortcuts:
5
+ 1. "JARVIS" Full voice assistant. Works on iPhone, iPad, Mac, Apple Watch.
6
+ Say "Hey Siri, JARVIS" → asks what you need → sends to HF Space → speaks result.
7
+ 2. "Ask JARVIS" — Quick text/voice input variant for Apple Watch.
8
 
9
+ Flow: Siri → Shortcut Dictate/text input POST to JARVIS server Speak result.
10
+
11
+ Works on ALL Apple devices because it hits the cloud server (HF Space / Render),
12
+ not localhost. The cloud server delegates macOS-specific tool calls to the
13
+ connected Mac listener via WebSocket.
14
  """
15
 
16
  import plistlib
17
  import subprocess
18
  import os
19
+ from dotenv import load_dotenv
20
+
21
+ load_dotenv()
22
 
23
  JARVIS_DIR = os.path.dirname(os.path.abspath(__file__))
24
+ HF_SPACE_URL = os.getenv("HF_SPACE_URL", "https://v1deh-jarvis.hf.space")
25
+ RENDER_URL = os.getenv("RENDER_URL", "")
26
+ LOCAL_TRIGGER_URL = "http://127.0.0.1:8111/trigger"
27
+
28
+ # Use cloud URL for cross-device, fall back to local
29
+ JARVIS_API_URL = RENDER_URL or HF_SPACE_URL
30
+ ASK_ENDPOINT = f"{JARVIS_API_URL}/api/ask"
31
+
32
+ # Auth token for the server (if any)
33
+ AUTH_TOKEN = os.getenv("JARVIS_AUTH_TOKEN", "")
34
+
35
+
36
+ def build_jarvis_shortcut() -> dict:
37
+ """Build the main JARVIS shortcut — voice in, voice out, all Apple devices."""
38
+ actions = [
39
+ # ── Action 1: Ask for voice input ("What do you need, sir?") ──
 
 
 
 
 
 
 
 
 
40
  {
41
+ "WFWorkflowActionIdentifier": "is.workflow.actions.ask",
42
  "WFWorkflowActionParameters": {
43
+ "WFAskActionPrompt": "What do you need, sir?",
44
+ "WFInputType": "Text",
45
  },
46
  },
47
+ # ── Action 2: Store the user's input ──
48
+ {
49
+ "WFWorkflowActionIdentifier": "is.workflow.actions.setvariable",
50
+ "WFWorkflowActionParameters": {
51
+ "WFVariableName": "UserCommand",
52
+ },
53
+ },
54
+ # ── Action 3: Build JSON body ──
55
+ {
56
+ "WFWorkflowActionIdentifier": "is.workflow.actions.gettext",
57
+ "WFWorkflowActionParameters": {
58
+ "WFTextActionText": {
59
+ "Value": {
60
+ "attachmentsByRange": {
61
+ "{0, 1}": {
62
+ "Type": "Variable",
63
+ "VariableName": "UserCommand",
64
+ },
65
+ },
66
+ "string": '{"message": "\uFFFC"}',
67
+ },
68
+ "WFSerializationType": "WFTextTokenString",
69
+ },
70
+ },
71
+ },
72
+ # ── Action 4: POST to JARVIS server ──
73
  {
74
  "WFWorkflowActionIdentifier": "is.workflow.actions.downloadurl",
75
  "WFWorkflowActionParameters": {
76
+ "WFURL": ASK_ENDPOINT,
77
+ "WFHTTPMethod": "POST",
78
+ "WFHTTPHeaders": {
79
+ "Value": {
80
+ "WFDictionaryFieldValueItems": [
81
+ {
82
+ "WFItemType": 0,
83
+ "WFKey": {"Value": {"string": "Content-Type"}, "WFSerializationType": "WFTextTokenString"},
84
+ "WFValue": {"Value": {"string": "application/json"}, "WFSerializationType": "WFTextTokenString"},
85
+ },
86
+ ] + ([
87
+ {
88
+ "WFItemType": 0,
89
+ "WFKey": {"Value": {"string": "Authorization"}, "WFSerializationType": "WFTextTokenString"},
90
+ "WFValue": {"Value": {"string": f"Bearer {AUTH_TOKEN}"}, "WFSerializationType": "WFTextTokenString"},
91
+ },
92
+ ] if AUTH_TOKEN else []),
93
+ },
94
+ "WFSerializationType": "WFDictionaryFieldValue",
95
+ },
96
+ "WFHTTPBodyType": "String",
97
+ "WFHTTPBodyString": {
98
+ "Value": {
99
+ "attachmentsByRange": {},
100
+ "string": "", # Will be set from previous action
101
+ },
102
+ "WFSerializationType": "WFTextTokenString",
103
+ },
104
+ },
105
+ },
106
+ # ── Action 5: Parse JSON response ──
107
+ {
108
+ "WFWorkflowActionIdentifier": "is.workflow.actions.detect.dictionary",
109
+ "WFWorkflowActionParameters": {},
110
+ },
111
+ # ── Action 6: Get "response" field from JSON ──
112
+ {
113
+ "WFWorkflowActionIdentifier": "is.workflow.actions.getvalueforkey",
114
+ "WFWorkflowActionParameters": {
115
+ "WFDictionaryKey": "response",
116
  },
117
  },
118
+ # ── Action 7: Speak the response aloud ──
119
+ {
120
+ "WFWorkflowActionIdentifier": "is.workflow.actions.speaktext",
121
+ "WFWorkflowActionParameters": {
122
+ "WFSpeakTextRate": 0.45, # Slightly faster
123
+ "WFSpeakTextWait": True,
124
+ },
125
+ },
126
+ # ── Action 8: Also show as notification ──
127
+ {
128
+ "WFWorkflowActionIdentifier": "is.workflow.actions.notification",
129
+ "WFWorkflowActionParameters": {
130
+ "WFNotificationActionTitle": "J.A.R.V.I.S.",
131
+ },
132
+ },
133
+ ]
134
+
135
+ # Also try to trigger local Mac listener (best-effort, won't fail on mobile)
136
+ actions.append({
137
+ "WFWorkflowActionIdentifier": "is.workflow.actions.downloadurl",
138
+ "WFWorkflowActionParameters": {
139
+ "WFURL": LOCAL_TRIGGER_URL,
140
+ "WFHTTPMethod": "GET",
141
+ },
142
+ })
143
+
144
+ return {
145
+ "WFWorkflowMinimumClientVersionString": "900",
146
+ "WFWorkflowMinimumClientVersion": 900,
147
+ "WFWorkflowClientVersion": "2302.0.4",
148
+ "WFWorkflowHasShortcutInputVariables": False,
149
+ "WFWorkflowImportQuestions": [],
150
+ "WFWorkflowName": "JARVIS",
151
+ "WFWorkflowIcon": {
152
+ "WFWorkflowIconStartColor": 463140863, # Blue
153
+ "WFWorkflowIconGlyphNumber": 59511, # Microphone icon
154
+ },
155
+ "WFWorkflowTypes": [
156
+ "NCWidget", # Notification Center
157
+ "WatchKit", # Apple Watch
158
+ ],
159
+ "WFWorkflowInputContentItemClasses": [
160
+ "WFStringContentItem",
161
+ ],
162
+ "WFWorkflowActions": actions,
163
+ }
164
+
165
+
166
+ def sign_and_install(shortcut_data: dict, name: str):
167
+ """Sign and install a shortcut."""
168
+ unsigned_path = os.path.join(JARVIS_DIR, f"{name}-unsigned.shortcut")
169
+ final_path = os.path.join(JARVIS_DIR, f"{name}.shortcut")
170
+
171
+ with open(unsigned_path, "wb") as f:
172
+ plistlib.dump(shortcut_data, f, fmt=plistlib.FMT_BINARY)
173
+
174
+ # Sign the shortcut
175
+ try:
176
+ subprocess.run(
177
+ ["shortcuts", "sign", "-m", "people-who-know-me",
178
+ "-i", unsigned_path, "-o", final_path],
179
+ check=True, capture_output=True,
180
+ )
181
+ os.remove(unsigned_path)
182
+ print(f" ✓ Signed: {final_path}")
183
+ except Exception as e:
184
+ print(f" [!] Signing failed: {e}")
185
+ os.rename(unsigned_path, final_path)
186
+ print(f" ✓ Unsigned: {final_path}")
187
+
188
+ # Auto-import
189
+ try:
190
+ result = subprocess.run(
191
+ ["open", final_path],
192
+ capture_output=True, text=True, timeout=10,
193
+ )
194
+ if result.returncode == 0:
195
+ print(f" → Shortcuts app should open — click 'Add Shortcut'")
196
+ else:
197
+ print(f" → Manual import: open '{final_path}'")
198
+ except Exception as e:
199
+ print(f" → Auto-import error: {e}")
200
+
201
+
202
+ if __name__ == "__main__":
203
+ print(f"JARVIS Server: {JARVIS_API_URL}")
204
+ print(f"Ask Endpoint: {ASK_ENDPOINT}")
205
+ if AUTH_TOKEN:
206
+ print(f"Auth Token: {AUTH_TOKEN[:8]}...")
207
+ print()
208
+
209
+ print("[1/1] Building JARVIS Siri Shortcut...")
210
+ shortcut = build_jarvis_shortcut()
211
+ sign_and_install(shortcut, "JARVIS")
212
+
213
+ print()
214
+ print("╔═══════════════════════════════════════════╗")
215
+ print("║ JARVIS Works On All Apple Devices: ║")
216
+ print("║ ║")
217
+ print('║ • Mac/iPhone/iPad: "Hey Siri, JARVIS" ║')
218
+ print("║ • Apple Watch: Raise wrist → 'JARVIS' ║")
219
+ print("║ • iPhone/iPad: PWA (Add to Home Screen) ║")
220
+ print("║ • Any device: Telegram bot ║")
221
+ print("║ ║")
222
+ print("║ To sync shortcut to iPhone/iPad/Watch: ║")
223
+ print("║ It syncs automatically via iCloud. ║")
224
+ print("║ Or AirDrop the .shortcut file. ║")
225
+ print("╚═══════════════════════════════════════════╝")
jarvis_app.py CHANGED
@@ -1,7 +1,8 @@
1
  """
2
  J.A.R.V.I.S. — macOS Native App with WebView UI.
3
 
4
- Shows the JARVIS web interface in a native window.
 
5
  No terminal. No browser. Just JARVIS.
6
  """
7
 
@@ -31,10 +32,31 @@ from AppKit import (
31
  from Foundation import NSObject, NSURL, NSURLRequest, NSMakeRect
32
  from WebKit import WKWebView, WKWebViewConfiguration, WKUserContentController
33
 
34
- JARVIS_URL = os.getenv("JARVIS_URL", "http://localhost:8000")
 
 
35
  JARVIS_DIR = os.path.dirname(os.path.abspath(__file__))
36
 
37
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
38
  class AppDelegate(NSObject):
39
  """Main application delegate — creates the WebView window."""
40
 
@@ -94,17 +116,40 @@ class AppDelegate(NSObject):
94
  NSApp.activateIgnoringOtherApps_(True)
95
 
96
  def _load_when_ready(self):
97
- """Wait for the JARVIS server to be available, then load the UI."""
 
98
  import httpx
99
 
100
- for i in range(30):
 
101
  try:
102
- httpx.get(f"{JARVIS_URL}/api/status", timeout=3)
 
103
  break
104
  except Exception:
105
  time.sleep(1)
106
 
107
- url = NSURL.URLWithString_(JARVIS_URL)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
108
  request = NSURLRequest.requestWithURL_(url)
109
  self.webview.performSelectorOnMainThread_withObject_waitUntilDone_(
110
  objc.selector(self.webview.loadRequest_, signature=b"v@:@"),
 
1
  """
2
  J.A.R.V.I.S. — macOS Native App with WebView UI.
3
 
4
+ Loads the JARVIS interface directly in a native WebKit window.
5
+ Tries local server first, then falls back to HuggingFace Space / Render.
6
  No terminal. No browser. Just JARVIS.
7
  """
8
 
 
32
  from Foundation import NSObject, NSURL, NSURLRequest, NSMakeRect
33
  from WebKit import WKWebView, WKWebViewConfiguration, WKUserContentController
34
 
35
+ JARVIS_LOCAL_URL = os.getenv("JARVIS_URL", "http://localhost:8000")
36
+ HF_SPACE_URL = os.getenv("HF_SPACE_URL", "https://v1deh-jarvis.hf.space")
37
+ RENDER_URL = os.getenv("RENDER_URL", "")
38
  JARVIS_DIR = os.path.dirname(os.path.abspath(__file__))
39
 
40
 
41
+ def _find_server() -> str:
42
+ """Try local → HF Space → Render. Return first reachable URL."""
43
+ import httpx
44
+ candidates = [JARVIS_LOCAL_URL, HF_SPACE_URL]
45
+ if RENDER_URL:
46
+ candidates.append(RENDER_URL)
47
+
48
+ for url in candidates:
49
+ if not url:
50
+ continue
51
+ try:
52
+ resp = httpx.get(f"{url}/api/status", timeout=5)
53
+ if resp.status_code == 200:
54
+ return url
55
+ except Exception:
56
+ continue
57
+ return ""
58
+
59
+
60
  class AppDelegate(NSObject):
61
  """Main application delegate — creates the WebView window."""
62
 
 
116
  NSApp.activateIgnoringOtherApps_(True)
117
 
118
  def _load_when_ready(self):
119
+ """Find reachable server (local or HF Space) and load the UI."""
120
+ # Try up to 15s for local server (it may be starting), then check remotes
121
  import httpx
122
 
123
+ target_url = ""
124
+ for i in range(15):
125
  try:
126
+ httpx.get(f"{JARVIS_LOCAL_URL}/api/status", timeout=3)
127
+ target_url = JARVIS_LOCAL_URL
128
  break
129
  except Exception:
130
  time.sleep(1)
131
 
132
+ # Local server not up — try HF Space / Render directly
133
+ if not target_url:
134
+ target_url = _find_server()
135
+
136
+ if not target_url:
137
+ # Nothing reachable — show error page in WebView
138
+ error_html = (
139
+ "data:text/html,"
140
+ "<html><body style='background:#0a0e17;color:#e0e6ed;font-family:monospace;"
141
+ "display:flex;align-items:center;justify-content:center;height:100vh;"
142
+ "flex-direction:column;'>"
143
+ "<h1 style='color:#00b4d8;letter-spacing:4px;'>J.A.R.V.I.S.</h1>"
144
+ "<p style='color:#8899aa;margin-top:16px;'>Cannot reach any server.</p>"
145
+ "<p style='color:#8899aa;font-size:12px;margin-top:8px;'>"
146
+ "Start local: <code>python server.py</code><br>"
147
+ "Or set HF_SPACE_URL / RENDER_URL in .env</p>"
148
+ "</body></html>"
149
+ )
150
+ target_url = error_html
151
+
152
+ url = NSURL.URLWithString_(target_url)
153
  request = NSURLRequest.requestWithURL_(url)
154
  self.webview.performSelectorOnMainThread_withObject_waitUntilDone_(
155
  objc.selector(self.webview.loadRequest_, signature=b"v@:@"),
jarvis_listener.py CHANGED
@@ -84,6 +84,81 @@ def touch_activity():
84
  _last_activity = time.time()
85
 
86
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
87
  def check_mic_permission() -> bool:
88
  """Test if microphone is actually returning audio data.
89
  macOS silently returns zeros when permission is not granted."""
@@ -495,6 +570,9 @@ EXIT_PHRASES = {"goodbye", "that's all", "thats all", "thank you", "thanks",
495
  "go to sleep", "nevermind", "never mind", "bye", "dismiss"}
496
  CONVERSATION_TIMEOUT = 30 # seconds of silence before auto-ending session
497
 
 
 
 
498
 
499
  def handle_activation(command_text=None):
500
  """
@@ -508,6 +586,8 @@ def handle_activation(command_text=None):
508
  return
509
 
510
  touch_activity()
 
 
511
 
512
  try:
513
  wake_pause_event.set()
@@ -532,8 +612,9 @@ def handle_activation(command_text=None):
532
 
533
  # Check if user wants to end conversation
534
  if command_text.lower().strip().rstrip(".!") in EXIT_PHRASES:
535
- speak("Standing by, sir.")
536
- log.info("Conversation ended by user")
 
537
  break
538
 
539
  # Process the command
@@ -543,6 +624,12 @@ def handle_activation(command_text=None):
543
  speak(answer)
544
 
545
  # Listen for follow-up (with shorter timeout)
 
 
 
 
 
 
546
  log.info("Listening for follow-up...")
547
  command_text = _listen_for_command()
548
  if not command_text:
@@ -643,6 +730,16 @@ def start_local_wake_listener():
643
  diag_max_amp = 0
644
 
645
  while True:
 
 
 
 
 
 
 
 
 
 
646
  if wake_pause_event.is_set():
647
  close_stream()
648
  clear_audio_state()
@@ -890,12 +987,10 @@ def server_watchdog():
890
  _device_id = None
891
 
892
 
893
- def device_heartbeat_loop():
894
- """Send periodic heartbeats to the server so the device shows as online."""
895
  global _device_id
896
  device_id_file = os.path.join(JARVIS_DIR, ".device_id")
897
-
898
- # Load or generate device ID
899
  if os.path.exists(device_id_file):
900
  with open(device_id_file) as f:
901
  _device_id = f.read().strip()
@@ -904,12 +999,17 @@ def device_heartbeat_loop():
904
  _device_id = str(uuid.uuid4())[:12]
905
  with open(device_id_file, "w") as f:
906
  f.write(_device_id)
 
 
907
 
 
 
 
908
  log.info(f"Device heartbeat: ID={_device_id}, interval=60s")
909
  while True:
910
  try:
911
  httpx.post(
912
- f"{JARVIS_URL}/api/device/heartbeat",
913
  json={"device_id": _device_id},
914
  timeout=5,
915
  )
@@ -918,6 +1018,96 @@ def device_heartbeat_loop():
918
  time.sleep(60)
919
 
920
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
921
  # ─── Clap/Snap Detection ─────────────────────────
922
  def start_clap_detector():
923
  """Start clap/snap detection as an additional wake trigger."""
@@ -975,17 +1165,44 @@ def run_menubar():
975
  self.title = "J"
976
  self.menu = [
977
  rumps.MenuItem("Activate JARVIS", callback=self.on_activate),
 
978
  None,
979
  rumps.MenuItem("Status", callback=self.on_status),
980
  None,
981
  rumps.MenuItem("Quit", callback=self.on_quit),
982
  ]
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
983
 
984
  def on_activate(self, _):
985
  touch_activity()
 
986
  threading.Thread(target=handle_activation, daemon=True).start()
987
  rumps.notification("J.A.R.V.I.S.", "", "Listening...", sound=False)
988
 
 
 
 
 
 
 
 
 
 
989
  def on_status(self, _):
990
  threading.Thread(target=self._check_status, daemon=True).start()
991
 
@@ -998,6 +1215,7 @@ def run_menubar():
998
  parts.append("Server: offline")
999
  parts.append(f"Wake: {ACTIVE_WAKE_MODE}")
1000
  parts.append(f"Clap: {'on' if ENABLE_CLAP_DETECTION else 'off'}")
 
1001
  rumps.notification("J.A.R.V.I.S. Status", "", " | ".join(parts))
1002
 
1003
  def on_quit(self, _):
@@ -1045,12 +1263,18 @@ def main():
1045
  threading.Thread(target=start_hotkey_listener, daemon=True).start()
1046
  threading.Thread(target=server_watchdog, daemon=True).start()
1047
  threading.Thread(target=device_heartbeat_loop, daemon=True).start()
 
1048
 
1049
  # Check mic permission before starting wake word listeners
1050
  mic_ok = check_mic_permission()
1051
  if not mic_ok:
1052
  log.warning("Mic check returned silent — will attempt wake listeners anyway (TCC may grant access to .app bundle)")
1053
 
 
 
 
 
 
1054
  if ACTIVE_WAKE_MODE == "local":
1055
  threading.Thread(target=start_local_wake_listener, daemon=True).start()
1056
 
 
84
  _last_activity = time.time()
85
 
86
 
87
+ # ─── Cached automation permission state ──────────
88
+ _automation_granted: dict[str, bool] = {}
89
+
90
+
91
+ def check_automation_permission(app_name: str) -> bool:
92
+ """Check if JARVIS has Automation permission for a given app.
93
+ Caches the result so macOS only prompts once per app per session."""
94
+ if app_name in _automation_granted:
95
+ return _automation_granted[app_name]
96
+
97
+ try:
98
+ result = subprocess.run(
99
+ ["osascript", "-e", f'tell application "{app_name}" to name'],
100
+ capture_output=True, text=True, timeout=10,
101
+ )
102
+ granted = result.returncode == 0
103
+ _automation_granted[app_name] = granted
104
+ if not granted:
105
+ log.warning(
106
+ f"Automation permission denied for {app_name}. "
107
+ f"Fix: System Settings → Privacy & Security → Automation → enable JARVIS for {app_name}"
108
+ )
109
+ return granted
110
+ except Exception as e:
111
+ log.warning(f"Automation permission check for {app_name} failed: {e}")
112
+ return False
113
+
114
+
115
+ def request_automation_permissions():
116
+ """Request Automation permissions for key apps at startup.
117
+ macOS will show a one-time prompt per app if not already granted.
118
+ This must run from the JARVIS process (not Terminal) so permissions
119
+ are associated with the JARVIS app bundle."""
120
+ apps_to_request = [
121
+ "System Events", # Required for UI automation, keystrokes, app control
122
+ "Finder", # File operations
123
+ "Calendar", # Calendar events
124
+ "Notes", # Note creation
125
+ "Reminders", # Reminders
126
+ "Mail", # Email
127
+ ]
128
+
129
+ log.info("Requesting Automation permissions for key apps...")
130
+ for app in apps_to_request:
131
+ granted = check_automation_permission(app)
132
+ if granted:
133
+ log.info(f" ✓ Automation: {app}")
134
+ else:
135
+ log.warning(f" ✗ Automation: {app} — user must grant in System Settings")
136
+ notify(
137
+ "JARVIS — Permission Needed",
138
+ f"Grant Automation access for {app}: System Settings → Privacy & Security → Automation",
139
+ )
140
+ # Small delay between prompts so the user sees each one
141
+ time.sleep(1)
142
+
143
+ # Check Accessibility (needed for global hotkey and UI control)
144
+ try:
145
+ result = subprocess.run(
146
+ ["osascript", "-e",
147
+ 'tell application "System Events" to key code 0'],
148
+ capture_output=True, text=True, timeout=5,
149
+ )
150
+ if result.returncode != 0:
151
+ log.warning("Accessibility access not granted.")
152
+ notify(
153
+ "JARVIS — Accessibility Needed",
154
+ "System Settings → Privacy & Security → Accessibility → enable JARVIS",
155
+ )
156
+ except Exception:
157
+ pass
158
+
159
+ log.info("Automation permission requests complete")
160
+
161
+
162
  def check_mic_permission() -> bool:
163
  """Test if microphone is actually returning audio data.
164
  macOS silently returns zeros when permission is not granted."""
 
570
  "go to sleep", "nevermind", "never mind", "bye", "dismiss"}
571
  CONVERSATION_TIMEOUT = 30 # seconds of silence before auto-ending session
572
 
573
+ # Standby mode — mic is fully off, only hotkey/Siri/menubar can re-activate
574
+ standby_mode = threading.Event()
575
+
576
 
577
  def handle_activation(command_text=None):
578
  """
 
586
  return
587
 
588
  touch_activity()
589
+ # Exit standby when user explicitly activates
590
+ standby_mode.clear()
591
 
592
  try:
593
  wake_pause_event.set()
 
612
 
613
  # Check if user wants to end conversation
614
  if command_text.lower().strip().rstrip(".!") in EXIT_PHRASES:
615
+ speak("Standing by, sir. Mic is off.")
616
+ log.info("Conversation ended by user — entering standby (mic off)")
617
+ standby_mode.set()
618
  break
619
 
620
  # Process the command
 
624
  speak(answer)
625
 
626
  # Listen for follow-up (with shorter timeout)
627
+ # If server returned an error, prompt user to retry
628
+ is_error = answer.startswith(("I can't reach", "The request timed out",
629
+ "I got a garbled", "I'm having trouble"))
630
+ if is_error:
631
+ speak("Would you like to try again, sir?")
632
+
633
  log.info("Listening for follow-up...")
634
  command_text = _listen_for_command()
635
  if not command_text:
 
730
  diag_max_amp = 0
731
 
732
  while True:
733
+ # Standby mode — mic fully off until hotkey/Siri/menubar reactivates
734
+ if standby_mode.is_set():
735
+ close_stream()
736
+ clear_audio_state()
737
+ diag_frame_count = 0
738
+ diag_speech_count = 0
739
+ diag_max_amp = 0
740
+ time.sleep(0.2)
741
+ continue
742
+
743
  if wake_pause_event.is_set():
744
  close_stream()
745
  clear_audio_state()
 
987
  _device_id = None
988
 
989
 
990
+ def _load_device_id():
991
+ """Load or generate a persistent device ID."""
992
  global _device_id
993
  device_id_file = os.path.join(JARVIS_DIR, ".device_id")
 
 
994
  if os.path.exists(device_id_file):
995
  with open(device_id_file) as f:
996
  _device_id = f.read().strip()
 
999
  _device_id = str(uuid.uuid4())[:12]
1000
  with open(device_id_file, "w") as f:
1001
  f.write(_device_id)
1002
+ return _device_id
1003
+
1004
 
1005
+ def device_heartbeat_loop():
1006
+ """Send periodic heartbeats to the server so the device shows as online."""
1007
+ _load_device_id()
1008
  log.info(f"Device heartbeat: ID={_device_id}, interval=60s")
1009
  while True:
1010
  try:
1011
  httpx.post(
1012
+ f"{JARVIS_URL}/api/devices/heartbeat",
1013
  json={"device_id": _device_id},
1014
  timeout=5,
1015
  )
 
1018
  time.sleep(60)
1019
 
1020
 
1021
+ def device_websocket_loop():
1022
+ """Connect to server via WebSocket to receive delegated tool commands in real-time.
1023
+
1024
+ When the JARVIS server runs on HF Space (Linux), macOS-only tools like
1025
+ open_app, set_volume, spotify_play, etc. get delegated to this connected
1026
+ device for local execution.
1027
+ """
1028
+ import websocket as ws_client
1029
+
1030
+ _load_device_id()
1031
+ ws_url = JARVIS_URL.replace("http://", "ws://").replace("https://", "wss://")
1032
+ endpoint = f"{ws_url}/ws/device/{_device_id}"
1033
+
1034
+ def _execute_tool_locally(tool_name: str, tool_args: dict) -> str:
1035
+ """Execute a tool on this Mac and return the result."""
1036
+ try:
1037
+ # Import tool registry (already loaded by server.py imports,
1038
+ # but listener runs standalone so we import directly)
1039
+ sys.path.insert(0, JARVIS_DIR)
1040
+ from tools import TOOL_REGISTRY, execute_tool as _exec
1041
+ import tools.builtin
1042
+ import tools.system_control
1043
+ import tools.app_automation
1044
+ import tools.vscode_tools
1045
+ import tools.device_control
1046
+
1047
+ if tool_name not in TOOL_REGISTRY:
1048
+ return f"Error: Unknown tool '{tool_name}'"
1049
+
1050
+ func = TOOL_REGISTRY[tool_name]["function"]
1051
+ result = func(**tool_args)
1052
+ # Handle async tools
1053
+ if hasattr(result, "__await__"):
1054
+ import asyncio
1055
+ loop = asyncio.new_event_loop()
1056
+ result = loop.run_until_complete(result)
1057
+ loop.close()
1058
+ return str(result)
1059
+ except Exception as e:
1060
+ log.error(f"Local tool execution failed: {tool_name}({tool_args}) -> {e}")
1061
+ return f"Error: {e}"
1062
+
1063
+ while True:
1064
+ try:
1065
+ log.info(f"Device WS: connecting to {endpoint}")
1066
+ ws = ws_client.WebSocket()
1067
+ ws.connect(endpoint, timeout=10)
1068
+ log.info(f"Device WS: connected as {_device_id}")
1069
+
1070
+ while True:
1071
+ msg = ws.recv()
1072
+ if not msg:
1073
+ continue
1074
+ data = json.loads(msg)
1075
+
1076
+ if data.get("type") == "command" and data.get("action") == "execute_tool":
1077
+ tool_name = data.get("tool", "")
1078
+ tool_args = data.get("args", {})
1079
+ log.info(f"Device WS: executing delegated tool {tool_name}({tool_args})")
1080
+ notify("JARVIS — Remote Command", f"Executing: {tool_name}")
1081
+
1082
+ result = _execute_tool_locally(tool_name, tool_args)
1083
+ log.info(f"Device WS: {tool_name} result: {result[:200]}")
1084
+
1085
+ # Report result back to server
1086
+ cmd_id = data.get("cmd_id", "")
1087
+ if cmd_id:
1088
+ ws.send(json.dumps({
1089
+ "type": "command_result",
1090
+ "cmd_id": cmd_id,
1091
+ "result": result,
1092
+ }))
1093
+ notify("JARVIS", f"{tool_name}: {result[:100]}")
1094
+
1095
+ elif data.get("type") == "command":
1096
+ # Generic command (not a tool call) — speak it
1097
+ command_text = data.get("command", "")
1098
+ if command_text:
1099
+ log.info(f"Device WS: received command: {command_text}")
1100
+ threading.Thread(
1101
+ target=handle_activation,
1102
+ kwargs={"command_text": command_text},
1103
+ daemon=True,
1104
+ ).start()
1105
+
1106
+ except Exception as e:
1107
+ log.warning(f"Device WS: connection lost ({e}), reconnecting in 10s...")
1108
+ time.sleep(10)
1109
+
1110
+
1111
  # ─── Clap/Snap Detection ─────────────────────────
1112
  def start_clap_detector():
1113
  """Start clap/snap detection as an additional wake trigger."""
 
1165
  self.title = "J"
1166
  self.menu = [
1167
  rumps.MenuItem("Activate JARVIS", callback=self.on_activate),
1168
+ rumps.MenuItem("Resume Listening", callback=self.on_resume),
1169
  None,
1170
  rumps.MenuItem("Status", callback=self.on_status),
1171
  None,
1172
  rumps.MenuItem("Quit", callback=self.on_quit),
1173
  ]
1174
+ # Monitor standby mode to update menu bar title
1175
+ threading.Thread(target=self._watch_standby, daemon=True).start()
1176
+
1177
+ def _watch_standby(self):
1178
+ """Update menu bar icon when entering/leaving standby."""
1179
+ was_standby = False
1180
+ while True:
1181
+ is_standby = standby_mode.is_set()
1182
+ if is_standby and not was_standby:
1183
+ self.title = "J💤"
1184
+ log.info("Menu bar: standby mode (mic off)")
1185
+ elif not is_standby and was_standby:
1186
+ self.title = "J"
1187
+ log.info("Menu bar: active mode")
1188
+ was_standby = is_standby
1189
+ time.sleep(1)
1190
 
1191
  def on_activate(self, _):
1192
  touch_activity()
1193
+ standby_mode.clear() # Exit standby on manual activation
1194
  threading.Thread(target=handle_activation, daemon=True).start()
1195
  rumps.notification("J.A.R.V.I.S.", "", "Listening...", sound=False)
1196
 
1197
+ def on_resume(self, _):
1198
+ """Resume wake word listening after standby."""
1199
+ if standby_mode.is_set():
1200
+ standby_mode.clear()
1201
+ rumps.notification("J.A.R.V.I.S.", "", "Wake word listening resumed", sound=True)
1202
+ speak("I'm back online, sir.")
1203
+ else:
1204
+ rumps.notification("J.A.R.V.I.S.", "", "Already listening", sound=False)
1205
+
1206
  def on_status(self, _):
1207
  threading.Thread(target=self._check_status, daemon=True).start()
1208
 
 
1215
  parts.append("Server: offline")
1216
  parts.append(f"Wake: {ACTIVE_WAKE_MODE}")
1217
  parts.append(f"Clap: {'on' if ENABLE_CLAP_DETECTION else 'off'}")
1218
+ parts.append(f"Mic: {'off (standby)' if standby_mode.is_set() else 'active'}")
1219
  rumps.notification("J.A.R.V.I.S. Status", "", " | ".join(parts))
1220
 
1221
  def on_quit(self, _):
 
1263
  threading.Thread(target=start_hotkey_listener, daemon=True).start()
1264
  threading.Thread(target=server_watchdog, daemon=True).start()
1265
  threading.Thread(target=device_heartbeat_loop, daemon=True).start()
1266
+ threading.Thread(target=device_websocket_loop, daemon=True).start()
1267
 
1268
  # Check mic permission before starting wake word listeners
1269
  mic_ok = check_mic_permission()
1270
  if not mic_ok:
1271
  log.warning("Mic check returned silent — will attempt wake listeners anyway (TCC may grant access to .app bundle)")
1272
 
1273
+ # Request Automation/Accessibility permissions at startup
1274
+ # This triggers macOS permission prompts from the JARVIS process
1275
+ # so permissions are associated with the JARVIS app bundle
1276
+ threading.Thread(target=request_automation_permissions, daemon=True).start()
1277
+
1278
  if ACTIVE_WAKE_MODE == "local":
1279
  threading.Thread(target=start_local_wake_listener, daemon=True).start()
1280
 
server.py CHANGED
@@ -33,7 +33,7 @@ load_dotenv()
33
  from memory import Memory
34
  from llm import SYSTEM_PROMPT, stream_response, get_active_backend, get_available_backends, FREE_MODELS, HF_FREE_MODELS
35
  from stm import apply_stm, AutoTune
36
- from tools import get_tools_prompt, execute_tool, TOOL_REGISTRY
37
  import tools.builtin # Register built-in tools
38
  import tools.system_control # System control tools
39
  import tools.user_tools # User profile, routines, work sessions
@@ -43,6 +43,9 @@ import tools.app_automation # Deep app control (Spotify, Notes, Calendar, Mail,
43
  import tools.device_onboarding # User device registration & cross-device commands
44
  from user_profile import get_user_context, get_preferences, get_today_routine, get_work_history, get_active_work_session
45
 
 
 
 
46
  # ── Auth Token ────────────────────────────────────────────────────
47
  # Set JARVIS_AUTH_TOKEN in .env for a fixed token; otherwise a random one is
48
  # generated each startup and printed to the console.
 
33
  from memory import Memory
34
  from llm import SYSTEM_PROMPT, stream_response, get_active_backend, get_available_backends, FREE_MODELS, HF_FREE_MODELS
35
  from stm import apply_stm, AutoTune
36
+ from tools import get_tools_prompt, execute_tool, TOOL_REGISTRY, register_macos_tools
37
  import tools.builtin # Register built-in tools
38
  import tools.system_control # System control tools
39
  import tools.user_tools # User profile, routines, work sessions
 
43
  import tools.device_onboarding # User device registration & cross-device commands
44
  from user_profile import get_user_context, get_preferences, get_today_routine, get_work_history, get_active_work_session
45
 
46
+ # Tag macOS-only tools so they get delegated to connected devices on Linux (HF Space)
47
+ register_macos_tools()
48
+
49
  # ── Auth Token ────────────────────────────────────────────────────
50
  # Set JARVIS_AUTH_TOKEN in .env for a fixed token; otherwise a random one is
51
  # generated each startup and printed to the console.
static/apple-touch-icon-152.png ADDED
static/apple-touch-icon-167.png ADDED
static/apple-touch-icon.png ADDED
static/index.html CHANGED
@@ -2,14 +2,21 @@
2
  <html lang="en">
3
  <head>
4
  <meta charset="UTF-8">
5
- <meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=no">
6
  <meta name="apple-mobile-web-app-capable" content="yes">
7
  <meta name="apple-mobile-web-app-status-bar-style" content="black-translucent">
8
  <meta name="apple-mobile-web-app-title" content="JARVIS">
9
  <meta name="mobile-web-app-capable" content="yes">
10
  <meta name="theme-color" content="#0a0e17">
 
11
  <link rel="manifest" href="/static/manifest.json">
12
- <link rel="apple-touch-icon" href="/static/icon-192.png">
 
 
 
 
 
 
13
  <title>J.A.R.V.I.S.</title>
14
  <style>
15
  * { margin: 0; padding: 0; box-sizing: border-box; }
@@ -482,6 +489,22 @@
482
  .welcome h2 { font-size: 20px; }
483
  .quick-actions { gap: 6px; }
484
  }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
485
  </style>
486
  </head>
487
  <body>
@@ -1159,28 +1182,41 @@
1159
  deferredPrompt = null;
1160
  }
1161
 
1162
- // iOS install hint
1163
  function checkiOSInstall() {
1164
- const isIOS = /iPad|iPhone|iPod/.test(navigator.userAgent);
 
1165
  const isStandalone = window.navigator.standalone;
1166
- if (isIOS && !isStandalone) {
 
 
 
 
1167
  const hint = document.createElement('div');
 
1168
  hint.style.cssText = `
1169
  position: fixed; bottom: 80px; left: 50%; transform: translateX(-50%);
1170
  background: var(--surface2); border: 1px solid var(--accent);
1171
- border-radius: 16px; padding: 12px 20px; z-index: 1000;
1172
  box-shadow: 0 0 30px var(--accent-glow); animation: fadeIn 0.5s ease;
1173
- max-width: 90vw; text-align: center; font-size: 12px; color: var(--text);
1174
  `;
1175
  hint.innerHTML = `
1176
- <div style="margin-bottom:6px; font-weight:bold; color: var(--accent);">Install JARVIS</div>
1177
- Tap <span style="font-size:16px;">&#9769;</span> Share → <strong>Add to Home Screen</strong>
1178
- <br><span style="color: var(--text2); font-size:10px;">Then it works like a real app — no browser needed</span>
1179
- <br><button onclick="this.parentElement.remove()" style="
1180
- margin-top:8px; background:none; border:1px solid var(--accent);
1181
- color:var(--accent); padding:4px 12px; border-radius:8px;
1182
- font-family:inherit; font-size:11px; cursor:pointer;
1183
- ">GOT IT</button>
 
 
 
 
 
 
 
1184
  `;
1185
  document.body.appendChild(hint);
1186
  }
 
2
  <html lang="en">
3
  <head>
4
  <meta charset="UTF-8">
5
+ <meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=no, viewport-fit=cover">
6
  <meta name="apple-mobile-web-app-capable" content="yes">
7
  <meta name="apple-mobile-web-app-status-bar-style" content="black-translucent">
8
  <meta name="apple-mobile-web-app-title" content="JARVIS">
9
  <meta name="mobile-web-app-capable" content="yes">
10
  <meta name="theme-color" content="#0a0e17">
11
+ <meta name="format-detection" content="telephone=no">
12
  <link rel="manifest" href="/static/manifest.json">
13
+ <!-- Apple touch icons for iPhone / iPad -->
14
+ <link rel="apple-touch-icon" href="/static/apple-touch-icon.png">
15
+ <link rel="apple-touch-icon" sizes="152x152" href="/static/apple-touch-icon-152.png">
16
+ <link rel="apple-touch-icon" sizes="167x167" href="/static/apple-touch-icon-167.png">
17
+ <link rel="apple-touch-icon" sizes="180x180" href="/static/apple-touch-icon.png">
18
+ <!-- iOS splash screens -->
19
+ <meta name="apple-mobile-web-app-orientations" content="portrait">
20
  <title>J.A.R.V.I.S.</title>
21
  <style>
22
  * { margin: 0; padding: 0; box-sizing: border-box; }
 
489
  .welcome h2 { font-size: 20px; }
490
  .quick-actions { gap: 6px; }
491
  }
492
+
493
+ /* iOS safe areas — notch / Dynamic Island / home indicator */
494
+ @supports (padding-top: env(safe-area-inset-top)) {
495
+ .header {
496
+ padding-top: calc(12px + env(safe-area-inset-top));
497
+ }
498
+ .input-area {
499
+ padding-bottom: calc(16px + env(safe-area-inset-bottom));
500
+ }
501
+ }
502
+
503
+ /* iPad specific — wider layout */
504
+ @media (min-width: 768px) and (max-width: 1024px) {
505
+ .message { max-width: 75%; }
506
+ .input-row { max-width: 700px; }
507
+ }
508
  </style>
509
  </head>
510
  <body>
 
1182
  deferredPrompt = null;
1183
  }
1184
 
1185
+ // iOS / iPadOS install hint
1186
  function checkiOSInstall() {
1187
+ const isIOS = /iPad|iPhone|iPod/.test(navigator.userAgent) ||
1188
+ (navigator.platform === 'MacIntel' && navigator.maxTouchPoints > 1); // iPad with desktop UA
1189
  const isStandalone = window.navigator.standalone;
1190
+ const dismissed = localStorage.getItem('jarvis_ios_install_dismissed');
1191
+ if (isIOS && !isStandalone && !dismissed) {
1192
+ const isIPad = /iPad/.test(navigator.userAgent) ||
1193
+ (navigator.platform === 'MacIntel' && navigator.maxTouchPoints > 1);
1194
+ const deviceName = isIPad ? 'iPad' : 'iPhone';
1195
  const hint = document.createElement('div');
1196
+ hint.id = 'ios-install-hint';
1197
  hint.style.cssText = `
1198
  position: fixed; bottom: 80px; left: 50%; transform: translateX(-50%);
1199
  background: var(--surface2); border: 1px solid var(--accent);
1200
+ border-radius: 16px; padding: 16px 24px; z-index: 1000;
1201
  box-shadow: 0 0 30px var(--accent-glow); animation: fadeIn 0.5s ease;
1202
+ max-width: 90vw; text-align: center; font-size: 13px; color: var(--text);
1203
  `;
1204
  hint.innerHTML = `
1205
+ <div style="margin-bottom:8px; font-weight:bold; color: var(--accent); font-size:15px;">Install JARVIS on ${deviceName}</div>
1206
+ <div style="margin-bottom:6px;">Tap <svg width="16" height="16" viewBox="0 0 24 24" fill="currentColor" style="vertical-align:middle;"><path d="M16 5l-1.42 1.42-1.59-1.59V16h-1.98V4.83L9.42 6.42 8 5l4-4 4 4zm4 5v11c0 1.1-.9 2-2 2H6c-1.11 0-2-.9-2-2V10c0-1.11.89-2 2-2h3v2H6v11h12V10h-3V8h3c1.1 0 2 .89 2 2z"/></svg> Share → <strong>Add to Home Screen</strong></div>
1207
+ <div style="color: var(--text2); font-size:11px; margin-bottom:10px;">JARVIS will run like a native app — fullscreen, with mic &amp; voice</div>
1208
+ <div style="display:flex; gap:8px; justify-content:center;">
1209
+ <button onclick="this.closest('#ios-install-hint').remove()" style="
1210
+ background:none; border:1px solid var(--accent);
1211
+ color:var(--accent); padding:6px 16px; border-radius:10px;
1212
+ font-family:inherit; font-size:12px; cursor:pointer;
1213
+ ">GOT IT</button>
1214
+ <button onclick="localStorage.setItem('jarvis_ios_install_dismissed','1'); this.closest('#ios-install-hint').remove()" style="
1215
+ background:none; border:1px solid var(--text2);
1216
+ color:var(--text2); padding:6px 16px; border-radius:10px;
1217
+ font-family:inherit; font-size:11px; cursor:pointer;
1218
+ ">DON'T SHOW AGAIN</button>
1219
+ </div>
1220
  `;
1221
  document.body.appendChild(hint);
1222
  }
static/manifest.json CHANGED
@@ -1,13 +1,32 @@
1
  {
2
  "name": "J.A.R.V.I.S.",
3
  "short_name": "JARVIS",
4
- "description": "Just A Rather Very Intelligent System",
5
  "start_url": "/",
6
  "display": "standalone",
7
  "background_color": "#0a0e17",
8
  "theme_color": "#00b4d8",
9
  "orientation": "portrait",
 
10
  "icons": [
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
11
  {
12
  "src": "/static/icon-192.png",
13
  "sizes": "192x192",
 
1
  {
2
  "name": "J.A.R.V.I.S.",
3
  "short_name": "JARVIS",
4
+ "description": "Just A Rather Very Intelligent System — Your AI Assistant",
5
  "start_url": "/",
6
  "display": "standalone",
7
  "background_color": "#0a0e17",
8
  "theme_color": "#00b4d8",
9
  "orientation": "portrait",
10
+ "scope": "/",
11
  "icons": [
12
+ {
13
+ "src": "/static/apple-touch-icon-152.png",
14
+ "sizes": "152x152",
15
+ "type": "image/png",
16
+ "purpose": "any"
17
+ },
18
+ {
19
+ "src": "/static/apple-touch-icon-167.png",
20
+ "sizes": "167x167",
21
+ "type": "image/png",
22
+ "purpose": "any"
23
+ },
24
+ {
25
+ "src": "/static/apple-touch-icon.png",
26
+ "sizes": "180x180",
27
+ "type": "image/png",
28
+ "purpose": "any"
29
+ },
30
  {
31
  "src": "/static/icon-192.png",
32
  "sizes": "192x192",
static/sw.js CHANGED
@@ -1,6 +1,14 @@
1
  // JARVIS Service Worker — enables offline + PWA install
2
- const CACHE_NAME = 'jarvis-v1';
3
- const ASSETS = ['/', '/static/manifest.json', '/static/icon-192.png', '/static/icon-512.png'];
 
 
 
 
 
 
 
 
4
 
5
  self.addEventListener('install', (event) => {
6
  event.waitUntil(
 
1
  // JARVIS Service Worker — enables offline + PWA install
2
+ const CACHE_NAME = 'jarvis-v2';
3
+ const ASSETS = [
4
+ '/',
5
+ '/static/manifest.json',
6
+ '/static/icon-192.png',
7
+ '/static/icon-512.png',
8
+ '/static/apple-touch-icon.png',
9
+ '/static/apple-touch-icon-152.png',
10
+ '/static/apple-touch-icon-167.png',
11
+ ];
12
 
13
  self.addEventListener('install', (event) => {
14
  event.waitUntil(
tests/test_app_automation.py CHANGED
@@ -161,20 +161,28 @@ class TestRemindersTools(unittest.TestCase):
161
 
162
  class TestCalendarTools(unittest.TestCase):
163
 
 
164
  @patch("tools.app_automation._osascript")
165
- def test_calendar_today(self, mock_osascript):
166
  mock_osascript.return_value = "• 10:00 AM — Team Standup\n• 2:00 PM — Design Review"
167
  from tools.app_automation import calendar_today
168
  result = calendar_today()
169
  self.assertIn("Standup", result)
170
 
 
171
  @patch("tools.app_automation._osascript")
172
- def test_calendar_create_event(self, mock_osascript):
173
  mock_osascript.return_value = "Event created: Lunch"
174
  from tools.app_automation import calendar_create_event
175
  result = calendar_create_event("Lunch", "2026-04-07 12:00", "2026-04-07 13:00", location="Cafe")
176
  self.assertIn("Event created", result)
177
 
 
 
 
 
 
 
178
 
179
  class TestBrowserTools(unittest.TestCase):
180
 
@@ -289,6 +297,46 @@ class TestUniversalAppControl(unittest.TestCase):
289
  self.assertIn("Times Square", result)
290
 
291
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
292
  class TestTotalToolCount(unittest.TestCase):
293
  """Ensure total tools meets target after all modules loaded."""
294
 
 
161
 
162
  class TestCalendarTools(unittest.TestCase):
163
 
164
+ @patch("tools.app_automation._check_calendar_permission", return_value=True)
165
  @patch("tools.app_automation._osascript")
166
+ def test_calendar_today(self, mock_osascript, mock_perm):
167
  mock_osascript.return_value = "• 10:00 AM — Team Standup\n• 2:00 PM — Design Review"
168
  from tools.app_automation import calendar_today
169
  result = calendar_today()
170
  self.assertIn("Standup", result)
171
 
172
+ @patch("tools.app_automation._check_calendar_permission", return_value=True)
173
  @patch("tools.app_automation._osascript")
174
+ def test_calendar_create_event(self, mock_osascript, mock_perm):
175
  mock_osascript.return_value = "Event created: Lunch"
176
  from tools.app_automation import calendar_create_event
177
  result = calendar_create_event("Lunch", "2026-04-07 12:00", "2026-04-07 13:00", location="Cafe")
178
  self.assertIn("Event created", result)
179
 
180
+ @patch("tools.app_automation._check_calendar_permission", return_value=False)
181
+ def test_calendar_permission_denied(self, mock_perm):
182
+ from tools.app_automation import calendar_today
183
+ result = calendar_today()
184
+ self.assertIn("Calendar access not granted", result)
185
+
186
 
187
  class TestBrowserTools(unittest.TestCase):
188
 
 
297
  self.assertIn("Times Square", result)
298
 
299
 
300
+ class TestOpenAppAliases(unittest.TestCase):
301
+ """Test open_app resolves common aliases to correct macOS app names."""
302
+
303
+ @patch("tools.builtin.subprocess.run")
304
+ def test_camera_alias(self, mock_run):
305
+ mock_run.return_value = MagicMock(returncode=0, stdout="", stderr="")
306
+ from tools.builtin import open_app
307
+ result = open_app("camera")
308
+ self.assertIn("FaceTime", result)
309
+ mock_run.assert_called_once_with(
310
+ ["open", "-a", "FaceTime"],
311
+ capture_output=True, text=True, timeout=10,
312
+ )
313
+
314
+ @patch("tools.builtin.subprocess.run")
315
+ def test_vscode_alias(self, mock_run):
316
+ mock_run.return_value = MagicMock(returncode=0, stdout="", stderr="")
317
+ from tools.builtin import open_app
318
+ result = open_app("vscode")
319
+ self.assertIn("Visual Studio Code", result)
320
+
321
+ @patch("tools.builtin.subprocess.run")
322
+ def test_direct_name_passthrough(self, mock_run):
323
+ mock_run.return_value = MagicMock(returncode=0, stdout="", stderr="")
324
+ from tools.builtin import open_app
325
+ result = open_app("Safari")
326
+ self.assertIn("Safari", result)
327
+
328
+ @patch("tools.builtin.subprocess.run")
329
+ def test_permission_error_message(self, mock_run):
330
+ mock_run.return_value = MagicMock(
331
+ returncode=1,
332
+ stdout="",
333
+ stderr="Not allowed to send Apple events",
334
+ )
335
+ from tools.builtin import open_app
336
+ result = open_app("Notes")
337
+ self.assertIn("Automation permission", result)
338
+
339
+
340
  class TestTotalToolCount(unittest.TestCase):
341
  """Ensure total tools meets target after all modules loaded."""
342
 
tools/__init__.py CHANGED
@@ -1,13 +1,25 @@
1
  """JARVIS Tool System — extensible tool registry."""
2
 
3
  import json
 
4
  from typing import Callable
5
 
6
  TOOL_REGISTRY: dict[str, dict] = {}
7
 
 
 
 
8
 
9
- def tool(name: str, description: str, parameters: dict = None):
10
- """Decorator to register a tool."""
 
 
 
 
 
 
 
 
11
  def decorator(func: Callable):
12
  TOOL_REGISTRY[name] = {
13
  "name": name,
@@ -15,13 +27,69 @@ def tool(name: str, description: str, parameters: dict = None):
15
  "parameters": parameters or {},
16
  "function": func,
17
  }
 
 
18
  return func
19
  return decorator
20
 
21
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
22
  async def execute_tool(name: str, args: dict) -> str:
23
  if name not in TOOL_REGISTRY:
24
  return f"Error: Unknown tool '{name}'"
 
 
 
 
 
25
  try:
26
  result = TOOL_REGISTRY[name]["function"](**args)
27
  if hasattr(result, "__await__"):
@@ -33,6 +101,48 @@ async def execute_tool(name: str, args: dict) -> str:
33
  return f"Error executing {name}: {type(e).__name__}"
34
 
35
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
36
  def get_tool_definitions() -> list[dict]:
37
  """Get tool definitions formatted for LLM function calling."""
38
  tools = []
 
1
  """JARVIS Tool System — extensible tool registry."""
2
 
3
  import json
4
+ import platform
5
  from typing import Callable
6
 
7
  TOOL_REGISTRY: dict[str, dict] = {}
8
 
9
+ # Tools that require macOS (osascript/subprocess) and should be delegated
10
+ # to a connected device when the server runs on Linux (HF Space / Render).
11
+ _MACOS_ONLY_TOOLS: set[str] = set()
12
 
13
+ IS_MACOS = platform.system() == "Darwin"
14
+
15
+
16
+ def tool(name: str, description: str, parameters: dict = None, macos_only: bool = False):
17
+ """Decorator to register a tool.
18
+
19
+ Args:
20
+ macos_only: If True, this tool requires macOS (osascript, AppleScript, etc.)
21
+ and will be delegated to a connected device when running on Linux.
22
+ """
23
  def decorator(func: Callable):
24
  TOOL_REGISTRY[name] = {
25
  "name": name,
 
27
  "parameters": parameters or {},
28
  "function": func,
29
  }
30
+ if macos_only:
31
+ _MACOS_ONLY_TOOLS.add(name)
32
  return func
33
  return decorator
34
 
35
 
36
+ async def _delegate_to_device(name: str, args: dict) -> str:
37
+ """Delegate a macOS-only tool call to a connected device via WebSocket push."""
38
+ try:
39
+ from user_device_registry import list_devices, send_command_to_device
40
+ except ImportError:
41
+ return f"Error: {name} requires macOS but this server runs on Linux. No device delegation available."
42
+
43
+ # Find an online device to delegate to
44
+ devices = await list_devices("default")
45
+ online_devices = [d for d in devices if d.get("status") == "online"]
46
+ if not online_devices:
47
+ return (
48
+ f"Error: '{name}' requires macOS but this server runs in the cloud. "
49
+ f"No connected Mac device found to execute this command. "
50
+ f"Make sure the JARVIS listener is running on your Mac."
51
+ )
52
+
53
+ target = online_devices[0]
54
+ device_id = target.get("device_id", "")
55
+ alias = target.get("alias", "device")
56
+
57
+ # Try WebSocket push first (instant)
58
+ try:
59
+ # Import at call time to avoid circular imports
60
+ import importlib
61
+ server_mod = importlib.import_module("server")
62
+ push_fn = getattr(server_mod, "push_command_to_device", None)
63
+ if push_fn:
64
+ pushed = await push_fn(device_id, {
65
+ "action": "execute_tool",
66
+ "tool": name,
67
+ "args": args,
68
+ })
69
+ if pushed:
70
+ return f"Command '{name}' sent to {alias} for execution."
71
+ except Exception:
72
+ pass
73
+
74
+ # Fallback: queue command via REST
75
+ result = await send_command_to_device(
76
+ target_alias=alias,
77
+ command=json.dumps({"tool": name, "args": args}),
78
+ user_id="default",
79
+ )
80
+ if "error" in result:
81
+ return f"Error delegating '{name}' to {alias}: {result['error']}"
82
+ return f"Command '{name}' queued for execution on {alias}."
83
+
84
+
85
  async def execute_tool(name: str, args: dict) -> str:
86
  if name not in TOOL_REGISTRY:
87
  return f"Error: Unknown tool '{name}'"
88
+
89
+ # If running on Linux (HF Space) and this tool needs macOS, delegate it
90
+ if not IS_MACOS and name in _MACOS_ONLY_TOOLS:
91
+ return await _delegate_to_device(name, args)
92
+
93
  try:
94
  result = TOOL_REGISTRY[name]["function"](**args)
95
  if hasattr(result, "__await__"):
 
101
  return f"Error executing {name}: {type(e).__name__}"
102
 
103
 
104
+ def register_macos_tools():
105
+ """Register all macOS-only tool names.
106
+
107
+ Called after all tool modules are imported so TOOL_REGISTRY is populated.
108
+ These tools use osascript, AppleScript, or macOS-specific CLI commands.
109
+ On Linux (HF Space / Render), they get automatically delegated to a
110
+ connected Mac device via WebSocket push.
111
+ """
112
+ macos_tools = {
113
+ # system_control.py — all 22 tools
114
+ "set_volume", "get_volume", "set_brightness", "toggle_dark_mode",
115
+ "wifi_control", "bluetooth_control", "do_not_disturb", "screenshot",
116
+ "media_control", "clipboard", "send_notification", "list_running_apps",
117
+ "quit_app", "set_timer", "lock_screen", "sleep_display", "empty_trash",
118
+ "announce", "open_url", "search_files", "create_reminder", "get_events",
119
+ # app_automation.py — all 36 tools
120
+ "spotify_play", "spotify_play_uri", "spotify_search", "spotify_queue",
121
+ "spotify_status", "spotify_control",
122
+ "notes_create", "notes_append", "notes_list", "notes_search",
123
+ "notes_read", "notes_delete",
124
+ "reminders_add", "reminders_list", "reminders_complete",
125
+ "calendar_create_event", "calendar_today",
126
+ "send_imessage", "read_messages",
127
+ "mail_compose", "mail_unread",
128
+ "browser_open", "browser_tabs", "browser_read_page",
129
+ "spotlight_search", "finder_open", "finder_move", "finder_copy", "trash_file",
130
+ "contacts_search", "maps_directions",
131
+ "app_keystroke", "app_menu_click", "app_window_manage", "textedit_create",
132
+ # builtin.py — macOS-specific subset
133
+ "open_app", "open_terminal",
134
+ # vscode_tools.py — all 7 tools
135
+ "vscode_open", "vscode_open_terminal", "vscode_run_command",
136
+ "copilot_chat", "copilot_inline", "vscode_list_extensions", "vscode_diff",
137
+ # device_control.py — macOS-specific subset
138
+ "bluetooth_scan", "bluetooth_pair", "bluetooth_connect",
139
+ "bluetooth_disconnect", "bluetooth_info", "network_scan",
140
+ "bonjour_discover", "homekit_control", "homekit_list_shortcuts",
141
+ "find_my_devices", "siri_command", "airdrop_file",
142
+ }
143
+ _MACOS_ONLY_TOOLS.update(macos_tools)
144
+
145
+
146
  def get_tool_definitions() -> list[dict]:
147
  """Get tool definitions formatted for LLM function calling."""
148
  tools = []
tools/app_automation.py CHANGED
@@ -44,6 +44,10 @@ def _osascript(script: str, timeout: int = 15) -> str:
44
  out = result.stdout.strip()
45
  if result.returncode != 0 and result.stderr.strip():
46
  err = result.stderr.strip()
 
 
 
 
47
  if "execution error" in err.lower():
48
  return f"Error: {err}"
49
  return out
@@ -605,6 +609,26 @@ def reminders_complete(title: str) -> str:
605
  # APPLE CALENDAR
606
  # ═══════════════════════════════════════════════════════════════
607
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
608
 
609
  @tool(
610
  name="calendar_create_event",
@@ -625,6 +649,9 @@ def reminders_complete(title: str) -> str:
625
  def calendar_create_event(title: str, start_date: str, end_date: str,
626
  location: str = "", notes: str = "",
627
  calendar_name: str = "") -> str:
 
 
 
628
  safe_title = title.replace('"', '\\"')
629
  safe_loc = location.replace('"', '\\"')
630
  safe_notes = notes.replace('"', '\\"')
@@ -654,6 +681,9 @@ def calendar_create_event(title: str, start_date: str, end_date: str,
654
  parameters={"type": "object", "properties": {}},
655
  )
656
  def calendar_today() -> str:
 
 
 
657
  result = _osascript('''
658
  set todayStart to current date
659
  set time of todayStart to 0
 
44
  out = result.stdout.strip()
45
  if result.returncode != 0 and result.stderr.strip():
46
  err = result.stderr.strip()
47
+ if "not allowed assistive access" in err.lower() or "not allowed to send keystrokes" in err.lower():
48
+ return ("Error: JARVIS needs Automation/Accessibility permission. "
49
+ "Go to System Settings → Privacy & Security → Automation and enable JARVIS, "
50
+ "then also check Accessibility.")
51
  if "execution error" in err.lower():
52
  return f"Error: {err}"
53
  return out
 
609
  # APPLE CALENDAR
610
  # ═══════════════════════════════════════════════════════════════
611
 
612
+ # Cache so Calendar permission is only checked once per session
613
+ _calendar_permission_checked = False
614
+ _calendar_permission_granted = False
615
+
616
+
617
+ def _check_calendar_permission() -> bool:
618
+ """Check Calendar automation permission once; cache the result."""
619
+ global _calendar_permission_checked, _calendar_permission_granted
620
+ if _calendar_permission_checked:
621
+ return _calendar_permission_granted
622
+ _calendar_permission_checked = True
623
+ result = subprocess.run(
624
+ ["osascript", "-e", 'tell application "Calendar" to name'],
625
+ capture_output=True, text=True, timeout=10,
626
+ )
627
+ _calendar_permission_granted = result.returncode == 0
628
+ if not _calendar_permission_granted:
629
+ _calendar_permission_granted = False
630
+ return _calendar_permission_granted
631
+
632
 
633
  @tool(
634
  name="calendar_create_event",
 
649
  def calendar_create_event(title: str, start_date: str, end_date: str,
650
  location: str = "", notes: str = "",
651
  calendar_name: str = "") -> str:
652
+ if not _check_calendar_permission():
653
+ return ("Error: Calendar access not granted. Go to System Settings → Privacy & Security → "
654
+ "Automation and allow JARVIS to control Calendar.")
655
  safe_title = title.replace('"', '\\"')
656
  safe_loc = location.replace('"', '\\"')
657
  safe_notes = notes.replace('"', '\\"')
 
681
  parameters={"type": "object", "properties": {}},
682
  )
683
  def calendar_today() -> str:
684
+ if not _check_calendar_permission():
685
+ return ("Error: Calendar access not granted. Go to System Settings → Privacy & Security → "
686
+ "Automation and allow JARVIS to control Calendar.")
687
  result = _osascript('''
688
  set todayStart to current date
689
  set time of todayStart to 0
tools/builtin.py CHANGED
@@ -367,21 +367,52 @@ def open_app(app_name: str) -> str:
367
  system = platform.system()
368
  try:
369
  if system == "Darwin":
370
- # Primary: use macOS `open -a`
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
371
  result = subprocess.run(
372
- ["open", "-a", app_name],
373
  capture_output=True, text=True, timeout=10,
374
  )
375
  if result.returncode == 0:
376
- return f"Opened {app_name}"
 
 
 
 
 
 
 
 
377
  # Fallback: AppleScript activate (handles apps `open -a` can't find)
378
  fallback = subprocess.run(
379
- ["osascript", "-e", f'tell application "{app_name}" to activate'],
380
  capture_output=True, text=True, timeout=10,
381
  )
382
  if fallback.returncode == 0:
383
- return f"Opened {app_name}"
384
- return f"Error opening {app_name}: {result.stderr.strip() or fallback.stderr.strip()}"
 
 
 
 
385
  elif system == "Linux":
386
  subprocess.Popen([app_name])
387
  elif system == "Windows":
 
367
  system = platform.system()
368
  try:
369
  if system == "Darwin":
370
+ # Map common aliases to actual macOS app names
371
+ app_aliases = {
372
+ "camera": "FaceTime",
373
+ "photo booth": "Photo Booth",
374
+ "facetime": "FaceTime",
375
+ "vscode": "Visual Studio Code",
376
+ "vs code": "Visual Studio Code",
377
+ "chrome": "Google Chrome",
378
+ "word": "Microsoft Word",
379
+ "excel": "Microsoft Excel",
380
+ "powerpoint": "Microsoft PowerPoint",
381
+ "teams": "Microsoft Teams",
382
+ "outlook": "Microsoft Outlook",
383
+ "code": "Visual Studio Code",
384
+ "iterm": "iTerm",
385
+ "terminal": "Terminal",
386
+ }
387
+ resolved_name = app_aliases.get(app_name.lower(), app_name)
388
+
389
+ # Primary: use macOS `open -a` (no Automation permission needed)
390
  result = subprocess.run(
391
+ ["open", "-a", resolved_name],
392
  capture_output=True, text=True, timeout=10,
393
  )
394
  if result.returncode == 0:
395
+ return f"Opened {resolved_name}"
396
+ # Retry with original name if alias didn't work
397
+ if resolved_name != app_name:
398
+ result2 = subprocess.run(
399
+ ["open", "-a", app_name],
400
+ capture_output=True, text=True, timeout=10,
401
+ )
402
+ if result2.returncode == 0:
403
+ return f"Opened {app_name}"
404
  # Fallback: AppleScript activate (handles apps `open -a` can't find)
405
  fallback = subprocess.run(
406
+ ["osascript", "-e", f'tell application "{resolved_name}" to activate'],
407
  capture_output=True, text=True, timeout=10,
408
  )
409
  if fallback.returncode == 0:
410
+ return f"Opened {resolved_name}"
411
+ err_msg = result.stderr.strip() or fallback.stderr.strip()
412
+ if "not allowed" in err_msg.lower():
413
+ return (f"Error opening {app_name}: JARVIS needs Automation permission. "
414
+ "Go to System Settings → Privacy & Security → Automation and enable JARVIS.")
415
+ return f"Error opening {app_name}: {err_msg}"
416
  elif system == "Linux":
417
  subprocess.Popen([app_name])
418
  elif system == "Windows":
uninstall.sh CHANGED
@@ -23,7 +23,12 @@ sleep 1
23
  # ─── Remove JARVIS .app bundles ───
24
  rm -rf "$HOME/jarvis/JARVIS-Listener.app"
25
  rm -rf "/Applications/JARVIS Listener.app"
26
- rm -rf "/Applications/JARVIS.app"
 
 
 
 
 
27
 
28
  # ─── Remove the Siri Shortcut ───
29
  # Delete the generated shortcut file
@@ -68,16 +73,16 @@ rm -rf "$HOME/jarvis/tools/__pycache__"
68
  tccutil reset Microphone ai.jarvis.listener 2>/dev/null || true
69
 
70
  echo ""
71
- echo " ╔═══════════════════════════════════════╗"
72
- echo " ║ J.A.R.V.I.S. UNINSTALLED ║"
73
- echo " ║ ║"
74
- echo " ║ ✓ Services stopped & removed ║"
75
- echo " ║ ✓ Mac apps removed ║"
76
- echo " ║ ✓ Siri Shortcut removed ║"
77
- echo " ║ ✓ Log files cleaned ║"
78
- echo " ║ ✓ Build artifacts removed ║"
79
- echo " ║ ║"
80
- echo " ║ Source code preserved at ~/jarvis/ ║"
81
  echo " ║ To reinstall: bash ~/jarvis/install.sh║"
82
- echo " ╚═══════════════════════════════════════╝"
83
  echo ""
 
23
  # ─── Remove JARVIS .app bundles ───
24
  rm -rf "$HOME/jarvis/JARVIS-Listener.app"
25
  rm -rf "/Applications/JARVIS Listener.app"
26
+ if [ -d "/Applications/JARVIS.app" ]; then
27
+ sudo rm -rf "/Applications/JARVIS.app" 2>/dev/null || {
28
+ echo " [!] Could not remove /Applications/JARVIS.app (need admin password)"
29
+ echo " [!] Run: sudo rm -rf /Applications/JARVIS.app"
30
+ }
31
+ fi
32
 
33
  # ─── Remove the Siri Shortcut ───
34
  # Delete the generated shortcut file
 
73
  tccutil reset Microphone ai.jarvis.listener 2>/dev/null || true
74
 
75
  echo ""
76
+ echo " ╔═════════════════════════════════════════╗"
77
+ echo " ║ J.A.R.V.I.S. UNINSTALLED ║"
78
+ echo " ║ ║"
79
+ echo " ║ ✓ Services stopped & removed ║"
80
+ echo " ║ ✓ Mac apps removed ║"
81
+ echo " ║ ✓ Siri Shortcut removed ║"
82
+ echo " ║ ✓ Log files cleaned ║"
83
+ echo " ║ ✓ Build artifacts removed ║"
84
+ echo " ║ ║"
85
+ echo " ║ Source code preserved at ~/jarvis/ ║"
86
  echo " ║ To reinstall: bash ~/jarvis/install.sh║"
87
+ echo " ╚═════════════════════════════════════════╝"
88
  echo ""