jarvis-cloud / modules /ar_interaction.py
Jarvis2345's picture
Upload folder using huggingface_hub
2dae9bb verified
Raw
History Blame Contribute Delete
4.83 kB
"""
modules/ar_interaction.py — FRIDAY AR Interaction Controls
Hand tracking + Mouse control for AR holograms.
"""
import os
import time
from typing import Optional, Tuple
from config import DATA_DIR
INTERACTION_FILE = os.path.join(DATA_DIR, "ar_interaction.json")
def _load() -> dict:
if not os.path.exists(INTERACTION_FILE):
return {"hand_enabled": False, "mouse_enabled": True, "last_hand_pos": None, "last_click": 0}
try:
import json
with open(INTERACTION_FILE, "r") as f:
return json.load(f)
except:
return {"hand_enabled": False, "mouse_enabled": True, "last_hand_pos": None, "last_click": 0}
def _save(data: dict) -> None:
try:
os.makedirs(DATA_DIR, exist_ok=True)
import json
with open(INTERACTION_FILE, "w") as f:
json.dump(data, f)
except:
pass
# === Hand Tracking ===
def enable_hand_tracking(enabled: bool = True) -> str:
"""Enable/disable hand tracking."""
data = _load()
data["hand_enabled"] = enabled
_save(data)
return f"Hand tracking {'enabled' if enabled else 'disabled'}."
def is_hand_tracking() -> bool:
return _load().get("hand_enabled", False)
def set_hand_position(x: float, y: float, z: float = 0.0) -> None:
"""Update hand position from tracking."""
data = _load()
data["last_hand_pos"] = {"x": x, "y": y, "z": z, "time": time.time()}
_save(data)
def get_hand_position() -> Optional[dict]:
"""Get last hand position."""
data = _load()
pos = data.get("last_hand_pos")
if pos and time.time() - pos.get("time", 0) < 0.5: # Fresh within 500ms
return pos
return None
# === Mouse Control ===
def enable_mouse_control(enabled: bool = True) -> str:
"""Enable/disable mouse control."""
data = _load()
data["mouse_enabled"] = enabled
_save(data)
return f"Mouse control {'enabled' if enabled else 'disabled'}."
def is_mouse_enabled() -> bool:
return _load().get("mouse_enabled", True)
def get_mouse_position() -> Tuple[int, int]:
"""Get current mouse position."""
try:
import ctypes
pt = ctypes.wintypes.POINT()
ctypes.windll.user32.GetCursorPos(ctypes.byref(pt))
return (pt.x, pt.y)
except:
return (0, 0)
def simulate_click() -> None:
"""Simulate a mouse click."""
try:
import ctypes
ctypes.windll.user32.mouse_event(0x0002, 0, 0, 0, 0) # MOUSEEVENTF_LEFTDOWN
ctypes.windll.user32.mouse_event(0x0004, 0, 0, 0, 0) # MOUSEEVENTF_LEFTUP
except:
pass
def simulate_right_click() -> None:
"""Simulate a right mouse click."""
try:
import ctypes
ctypes.windll.user32.mouse_event(0x0008, 0, 0, 0, 0) # MOUSEEVENTF_RIGHTDOWN
ctypes.windll.user32.mouse32mouse_event(0x0010, 0, 0, 0, 0) # MOUSEEVENTF_RIGHTUP
except:
pass
def drag_to(x: int, y: int) -> None:
"""Drag mouse to position."""
try:
import ctypes
ctypes.windll.user32.SetCursorPos(x, y)
except:
pass
# === AR Interaction State ===
def get_interaction_state() -> dict:
"""Get current interaction state for AR."""
hand_pos = get_hand_position()
mouse_pos = get_mouse_position()
return {
"hand_enabled": is_hand_tracking(),
"mouse_enabled": is_mouse_enabled(),
"hand_position": hand_pos,
"mouse_position": mouse_pos,
}
def process_gesture(gesture: str) -> str:
"""Process hand gesture commands."""
gesture = gesture.lower()
if "fist" in gesture or "grab" in gesture:
simulate_click()
return "Grab - Clicked!"
elif "point" in gesture:
return "Pointing..."
elif "wave" in gesture:
return "Waving..."
elif "open" in gesture:
return "Hand open..."
else:
return f"Gesture: {gesture}"
# === Voice Commands ===
def handle_command(command: str, speak) -> bool:
c = command.lower()
if "hand track" in c:
if "enable" in c or "on" in c:
result = enable_hand_tracking(True)
elif "disable" in c or "off" in c:
result = enable_hand_tracking(False)
else:
result = "Hand tracking enabled." if enable_hand_tracking(True) else ""
speak(result)
return True
if "mouse control" in c:
if "enable" in c or "on" in c:
result = enable_mouse_control(True)
elif "disable" in c or "off" in c:
result = enable_mouse_control(False)
else:
result = "Mouse control enabled."
speak(result)
return True
return False