"""Déclarations outils Gemini Live — port Mark-XLVIII.""" from __future__ import annotations _SKIP_SKILL = { "skill_loader", "_base", "proactive", "fallback_agent", "dev_agent_skill", "screen_processor", "system_monitor_skill", } def skill_tool_declaration() -> dict: from emo.desktop.actions.skill_loader import list_skills names = [n for n in list_skills() if n not in _SKIP_SKILL] hint = ", ".join(names[:20]) if names else "aucun" return { "name": "run_custom_skill", "description": ( "Exécute un skill Python du dossier actions/. " f"Skills disponibles : {hint}. " "Pour imprimer un fichier : skill_name=print_document, query=chemin complet. " "Pour lancer un .py : skill_name=run_python_script, query=chemin du script." ), "parameters": { "type": "OBJECT", "properties": { "skill_name": { "type": "STRING", "description": "Nom du fichier skill sans .py", }, "query": { "type": "STRING", "description": "Requête ou paramètre principal pour le skill", }, }, "required": ["skill_name"], }, } _TOOL_DECLARATIONS = [ { "name": "open_app", "description": ( "Opens any application on the computer. " "Use this whenever the user asks to open, launch, or start any app, " "website, or program. Always call this tool — never just say you opened it." ), "parameters": { "type": "OBJECT", "properties": { "app_name": { "type": "STRING", "description": "Exact name of the application (e.g. 'WhatsApp', 'Chrome', 'Spotify')" } }, "required": ["app_name"] } }, { "name": "web_search", "description": ( "Searches the web. Use for ANY question about current facts, events, prices, " "or topics — always prefer this over guessing. " "Modes: 'search' (default), 'news' (latest headlines on a topic), " "'research' (deep comprehensive answer), 'price' (product cost lookup), " "'compare' (side-by-side comparison of items)." ), "parameters": { "type": "OBJECT", "properties": { "query": {"type": "STRING", "description": "Search query or topic"}, "mode": {"type": "STRING", "description": "search | news | research | price | compare"}, "items": {"type": "ARRAY", "items": {"type": "STRING"}, "description": "Items to compare (compare mode)"}, "aspect": {"type": "STRING", "description": "Comparison aspect: price | specs | reviews | features"}, }, "required": ["query"] } }, { "name": "system_monitor_skill", "description": ( "Returns real-time system metrics: CPU usage, RAM, GPU load, CPU temperature, " "uptime, and process count. Use when the user asks about computer performance, " "temperature, memory, or resource usage." ), "parameters": { "type": "OBJECT", "properties": {}, } }, { "name": "weather_report", "description": "Gives the weather report to user", "parameters": { "type": "OBJECT", "properties": { "city": {"type": "STRING", "description": "City name"} }, "required": ["city"] } }, { "name": "send_message", "description": "Sends a text message via WhatsApp, Telegram, or other messaging platform.", "parameters": { "type": "OBJECT", "properties": { "receiver": {"type": "STRING", "description": "Recipient contact name"}, "message_text": {"type": "STRING", "description": "The message to send"}, "platform": {"type": "STRING", "description": "Platform: WhatsApp, Telegram, etc."} }, "required": ["receiver", "message_text", "platform"] } }, { "name": "reminder", "description": "Sets a timed reminder using Task Scheduler.", "parameters": { "type": "OBJECT", "properties": { "date": {"type": "STRING", "description": "Date in YYYY-MM-DD format"}, "time": {"type": "STRING", "description": "Time in HH:MM format (24h)"}, "message": {"type": "STRING", "description": "Reminder message text"} }, "required": ["date", "time", "message"] } }, { "name": "youtube_video", "description": ( "Controls YouTube. Use for: playing videos, summarizing a video's content, " "getting video info, or showing trending videos." ), "parameters": { "type": "OBJECT", "properties": { "action": {"type": "STRING", "description": "play | summarize | get_info | trending (default: play)"}, "query": {"type": "STRING", "description": "Search query for play action"}, "save": {"type": "BOOLEAN", "description": "Save summary to Notepad (summarize only)"}, "region": {"type": "STRING", "description": "Country code for trending e.g. TR, US"}, "url": {"type": "STRING", "description": "Video URL for get_info action"}, }, "required": [] } }, { "name": "screen_processor", "description": ( "Captures the screen or webcam image and lets you analyze it. " "MUST be called when user asks what is on screen, what you see, " "look at camera, analyze my screen, etc. " "You have NO visual ability without this tool. " "After the image is captured it is sent directly to you — describe what you see and answer the user's question. " "When using camera: the live view stays open until user says close it or calls close_camera." ), "parameters": { "type": "OBJECT", "properties": { "angle": {"type": "STRING", "description": "'screen' to capture display, 'camera' for webcam. Default: 'screen'"}, "text": {"type": "STRING", "description": "The question or instruction about the captured image"} }, "required": ["text"] } }, { "name": "close_camera", "description": ( "Closes the live camera view shown on screen. " "Call when user says: close camera, stop camera, turn off camera, " "kamerayı kapat, kapat, creepy, etc." ), "parameters": {"type": "OBJECT", "properties": {}, "required": []} }, { "name": "computer_settings", "description": ( "Controls the computer: volume, brightness, window management, keyboard shortcuts, " "typing text on screen, closing apps, fullscreen, dark mode, WiFi, restart, shutdown, " "scrolling, tab management, zoom, screenshots, lock screen, refresh/reload page. " "Use for ANY single computer control command." ), "parameters": { "type": "OBJECT", "properties": { "action": {"type": "STRING", "description": "The action to perform"}, "description": {"type": "STRING", "description": "Natural language description of what to do"}, "value": {"type": "STRING", "description": "Optional value: volume level, text to type, etc."} }, "required": [] } }, { "name": "browser_control", "description": ( "Controls any web browser. Use for: opening websites, searching the web, " "clicking elements, filling forms, scrolling, screenshots, navigation, any web-based task. " "Always pass the 'browser' parameter when the user specifies a browser (e.g. 'open in Edge', " "'use Firefox', 'open Chrome'). Multiple browsers can run simultaneously." ), "parameters": { "type": "OBJECT", "properties": { "action": {"type": "STRING", "description": "go_to | search | click | type | scroll | fill_form | smart_click | smart_type | get_text | get_url | press | new_tab | close_tab | screenshot | back | forward | reload | switch | list_browsers | close | close_all"}, "browser": {"type": "STRING", "description": "Target browser: chrome | edge | firefox | opera | operagx | brave | vivaldi | safari. Omit to use the currently active browser."}, "url": {"type": "STRING", "description": "URL for go_to / new_tab action"}, "query": {"type": "STRING", "description": "Search query for search action"}, "engine": {"type": "STRING", "description": "Search engine: google | bing | duckduckgo | yandex (default: google)"}, "selector": {"type": "STRING", "description": "CSS selector for click/type"}, "text": {"type": "STRING", "description": "Text to click or type"}, "description": {"type": "STRING", "description": "Element description for smart_click/smart_type"}, "direction": {"type": "STRING", "description": "up | down for scroll"}, "amount": {"type": "INTEGER", "description": "Scroll amount in pixels (default: 500)"}, "key": {"type": "STRING", "description": "Key name for press action (e.g. Enter, Escape, F5)"}, "path": {"type": "STRING", "description": "Save path for screenshot"}, "incognito": {"type": "BOOLEAN", "description": "Open in private/incognito mode"}, "clear_first": {"type": "BOOLEAN", "description": "Clear field before typing (default: true)"}, }, "required": ["action"] } }, { "name": "file_controller", "description": "Manages files and folders: list, create, delete, move, copy, rename, read, write, find, disk usage.", "parameters": { "type": "OBJECT", "properties": { "action": {"type": "STRING", "description": "list | create_file | create_folder | delete | move | copy | rename | read | write | find | largest | disk_usage | organize_desktop | info"}, "path": {"type": "STRING", "description": "File/folder path or shortcut: desktop, downloads, documents, home"}, "destination": {"type": "STRING", "description": "Destination path for move/copy"}, "new_name": {"type": "STRING", "description": "New name for rename"}, "content": {"type": "STRING", "description": "Content for create_file/write"}, "name": {"type": "STRING", "description": "File name to search for"}, "extension": {"type": "STRING", "description": "File extension to search (e.g. .pdf)"}, "count": {"type": "INTEGER", "description": "Number of results for largest"}, }, "required": ["action"] } }, { "name": "desktop_control", "description": "Controls the desktop: wallpaper, organize, clean, list, stats.", "parameters": { "type": "OBJECT", "properties": { "action": {"type": "STRING", "description": "wallpaper | wallpaper_url | organize | clean | list | stats | task"}, "path": {"type": "STRING", "description": "Image path for wallpaper"}, "url": {"type": "STRING", "description": "Image URL for wallpaper_url"}, "mode": {"type": "STRING", "description": "by_type or by_date for organize"}, "task": {"type": "STRING", "description": "Natural language desktop task"}, }, "required": ["action"] } }, { "name": "code_helper", "description": "Writes, edits, explains, runs, or builds code files.", "parameters": { "type": "OBJECT", "properties": { "action": {"type": "STRING", "description": "write | edit | explain | run | build | auto (default: auto)"}, "description": {"type": "STRING", "description": "What the code should do or what change to make"}, "language": {"type": "STRING", "description": "Programming language (default: python)"}, "output_path": {"type": "STRING", "description": "Where to save the file"}, "file_path": {"type": "STRING", "description": "Path to existing file for edit/explain/run/build"}, "code": {"type": "STRING", "description": "Raw code string for explain"}, "args": {"type": "STRING", "description": "CLI arguments for run/build"}, "timeout": {"type": "INTEGER", "description": "Execution timeout in seconds (default: 30)"}, }, "required": ["action"] } }, { "name": "dev_agent_skill", "description": ( "Construit des projets multi-fichiers — petits ou MEGA (launcher, plateforme, SaaS). " "Détecte automatiquement l'échelle : architecture multi-domaines, phases, marketplace, " "auth, instances, API, UI, etc. " "Pour un gros projet : mode analyze d'abord (plan complet), puis execute." ), "parameters": { "type": "OBJECT", "properties": { "description": {"type": "STRING", "description": "What the project should do — décrivez TOUTES les fonctionnalités"}, "language": {"type": "STRING", "description": "Programming language (default: python)"}, "project_name": {"type": "STRING", "description": "Optional project folder name"}, "mode": {"type": "STRING", "description": "analyze (plan architecture) | execute (build complet)"}, "timeout": {"type": "INTEGER", "description": "Run timeout in seconds (default: 30)"}, }, "required": ["description"] } }, { "name": "fallback_agent", "description": ( "Routeur intelligent local : Emo décide automatiquement s'il faut exécuter une skill, " "en créer une nouvelle, ou simplement répondre. " "Passez la demande telle quelle — pas besoin de préciser « skill » ou le nom de l'outil. " "La création de skill s'exécute en arrière-plan." ), "parameters": { "type": "OBJECT", "properties": { "task": {"type": "STRING", "description": "The developer or skill request to solve."}, "context": {"type": "STRING", "description": "Optional context, error output, or file list."}, "folder": {"type": "STRING", "description": "Project folder path (Agent Mode)."}, }, "required": ["task"] } }, { "name": "computer_control", "description": "Direct computer control: type, click, hotkeys, scroll, move mouse, screenshots, find elements on screen.", "parameters": { "type": "OBJECT", "properties": { "action": {"type": "STRING", "description": "type | smart_type | click | double_click | right_click | hotkey | press | scroll | move | copy | paste | screenshot | wait | clear_field | focus_window | screen_find | screen_click | random_data | user_data"}, "text": {"type": "STRING", "description": "Text to type or paste"}, "x": {"type": "INTEGER", "description": "X coordinate"}, "y": {"type": "INTEGER", "description": "Y coordinate"}, "keys": {"type": "STRING", "description": "Key combination e.g. 'ctrl+c'"}, "key": {"type": "STRING", "description": "Single key e.g. 'enter'"}, "direction": {"type": "STRING", "description": "up | down | left | right"}, "amount": {"type": "INTEGER", "description": "Scroll amount (default: 3)"}, "seconds": {"type": "NUMBER", "description": "Seconds to wait"}, "title": {"type": "STRING", "description": "Window title for focus_window"}, "description": {"type": "STRING", "description": "Element description for screen_find/screen_click"}, "type": {"type": "STRING", "description": "Data type for random_data"}, "field": {"type": "STRING", "description": "Field for user_data: name|email|city"}, "clear_first": {"type": "BOOLEAN", "description": "Clear field before typing (default: true)"}, "path": {"type": "STRING", "description": "Save path for screenshot"}, }, "required": ["action"] } }, { "name": "game_updater", "description": ( "THE ONLY tool for ANY Steam or Epic Games request. " "Use for: installing, downloading, updating games, listing installed games, " "checking download status, scheduling updates. " "ALWAYS call directly for any Steam/Epic/game request. " "NEVER use browser_control or web_search for Steam/Epic." ), "parameters": { "type": "OBJECT", "properties": { "action": {"type": "STRING", "description": "update | install | list | download_status | schedule | cancel_schedule | schedule_status (default: update)"}, "platform": {"type": "STRING", "description": "steam | epic | both (default: both)"}, "game_name": {"type": "STRING", "description": "Game name (partial match supported)"}, "app_id": {"type": "STRING", "description": "Steam AppID for install (optional)"}, "hour": {"type": "INTEGER", "description": "Hour for scheduled update 0-23 (default: 3)"}, "minute": {"type": "INTEGER", "description": "Minute for scheduled update 0-59 (default: 0)"}, "shutdown_when_done": {"type": "BOOLEAN", "description": "Shut down PC when download finishes"}, }, "required": [] } }, { "name": "flight_finder", "description": "Searches Google Flights and speaks the best options.", "parameters": { "type": "OBJECT", "properties": { "origin": {"type": "STRING", "description": "Departure city or airport code"}, "destination": {"type": "STRING", "description": "Arrival city or airport code"}, "date": {"type": "STRING", "description": "Departure date (any format)"}, "return_date": {"type": "STRING", "description": "Return date for round trips"}, "passengers": {"type": "INTEGER", "description": "Number of passengers (default: 1)"}, "cabin": {"type": "STRING", "description": "economy | premium | business | first"}, "save": {"type": "BOOLEAN", "description": "Save results to Notepad"}, }, "required": ["origin", "destination", "date"] } }, { "name": "shutdown_jarvis", "description": ( "Shuts down the assistant completely. " "Call this when the user expresses intent to end the conversation, " "close the assistant, say goodbye, or stop Jarvis. " "The user can say this in ANY language." ), "parameters": { "type": "OBJECT", "properties": {}, } }, { "name": "file_processor", "description": ( "Processes any file that the user has uploaded or dropped onto the interface. " "Use this when the user refers to an uploaded file and wants an action on it. " "Supports: images (describe/ocr/resize/compress/convert), " "PDFs (summarize/extract_text/to_word), " "Word docs & text files (summarize/fix/reformat/translate), " "CSV/Excel (analyze/stats/filter/sort/convert), " "JSON/XML (validate/format/analyze), " "code files (explain/review/fix/optimize/run/document/test), " "audio (transcribe/trim/convert/info), " "video (trim/extract_audio/extract_frame/compress/transcribe/info), " "archives (list/extract), " "presentations (summarize/extract_text). " "ALWAYS call this tool when a file has been uploaded and the user gives a command about it. " "If the user's command is ambiguous, pick the most logical action for that file type." ), "parameters": { "type": "OBJECT", "properties": { "file_path": { "type": "STRING", "description": "Full path to the uploaded file. Leave empty to use the currently uploaded file." }, "action": { "type": "STRING", "description": ( "What to do with the file. Examples by type:\n" "image: describe | ocr | resize | compress | convert | info\n" "pdf: summarize | extract_text | to_word | info\n" "docx/txt: summarize | fix | reformat | translate_hint | word_count | to_bullet\n" "csv/excel: analyze | stats | filter | sort | convert | info\n" "json: validate | format | analyze | to_csv\n" "code: explain | review | fix | optimize | run | document | test\n" "audio: transcribe | trim | convert | info\n" "video: trim | extract_audio | extract_frame | compress | transcribe | info | convert\n" "archive: list | extract\n" "pptx: summarize | extract_text | analyze" ) }, "instruction": { "type": "STRING", "description": "Free-form instruction if action doesn't cover it. E.g. 'translate this to Turkish', 'find all email addresses'" }, "format": { "type": "STRING", "description": "Target format for conversion. E.g. 'mp3', 'pdf', 'csv', 'png'" }, "width": {"type": "INTEGER", "description": "Target width for image resize"}, "height": {"type": "INTEGER", "description": "Target height for image resize"}, "scale": {"type": "NUMBER", "description": "Scale factor for image resize (e.g. 0.5)"}, "quality": {"type": "INTEGER", "description": "Quality 1-100 for image/video compress"}, "start": {"type": "STRING", "description": "Start time for trim: seconds or HH:MM:SS"}, "end": {"type": "STRING", "description": "End time for trim: seconds or HH:MM:SS"}, "timestamp": {"type": "STRING", "description": "Timestamp for video frame extraction HH:MM:SS"}, "column": {"type": "STRING", "description": "Column name for CSV filter/sort"}, "value": {"type": "STRING", "description": "Filter value for CSV filter"}, "condition": {"type": "STRING", "description": "Filter condition: equals|contains|gt|lt"}, "ascending": {"type": "BOOLEAN", "description": "Sort order for CSV sort (default: true)"}, "save": {"type": "BOOLEAN", "description": "Save result to file (default: true)"}, "destination": {"type": "STRING", "description": "Output folder for archive extract"}, }, "required": [] } }, { "name": "save_memory", "description": ( "Save an important personal fact about the user to long-term memory. " "Call this silently whenever the user reveals something worth remembering: " "name, age, city, job, preferences, hobbies, relationships, projects, or future plans. " "Do NOT call for: weather, reminders, searches, or one-time commands. " "Do NOT announce that you are saving — just call it silently. " "Values must be in English regardless of the conversation language." ), "parameters": { "type": "OBJECT", "properties": { "category": { "type": "STRING", "description": ( "identity — name, age, birthday, city, job, language, nationality | " "preferences — favorite food/color/music/film/game/sport, hobbies | " "projects — active projects, goals, things being built | " "relationships — friends, family, partner, colleagues | " "wishes — future plans, things to buy, travel dreams | " "notes — habits, schedule, anything else worth remembering" ) }, "key": {"type": "STRING", "description": "Short snake_case key (e.g. name, favorite_food, sister_name)"}, "value": {"type": "STRING", "description": "Concise value in English (e.g. Fatih, pizza, older sister)"}, }, "required": ["category", "key", "value"] } }, ] def get_tool_declarations() -> list[dict]: decls = list(_TOOL_DECLARATIONS) decls.append(skill_tool_declaration()) return decls