Spaces:
Sleeping
Sleeping
| """Déclarations outils Gemini Live — port Mark-XLVIII.""" | |
| from __future__ import annotations | |
| _SKIP_SKILL = { | |
| "skill_loader", "_base", "proactive", "fallback_agent", | |
| "dev_agent_skill", "screen_processor", "system_monitor_skill", | |
| } | |
| def skill_tool_declaration() -> dict: | |
| from emo.desktop.actions.skill_loader import list_skills | |
| names = [n for n in list_skills() if n not in _SKIP_SKILL] | |
| hint = ", ".join(names[:20]) if names else "aucun" | |
| return { | |
| "name": "run_custom_skill", | |
| "description": ( | |
| "Exécute un skill Python du dossier actions/. " | |
| f"Skills disponibles : {hint}. " | |
| "Pour imprimer un fichier : skill_name=print_document, query=chemin complet. " | |
| "Pour lancer un .py : skill_name=run_python_script, query=chemin du script." | |
| ), | |
| "parameters": { | |
| "type": "OBJECT", | |
| "properties": { | |
| "skill_name": { | |
| "type": "STRING", | |
| "description": "Nom du fichier skill sans .py", | |
| }, | |
| "query": { | |
| "type": "STRING", | |
| "description": "Requête ou paramètre principal pour le skill", | |
| }, | |
| }, | |
| "required": ["skill_name"], | |
| }, | |
| } | |
| _TOOL_DECLARATIONS = [ | |
| { | |
| "name": "open_app", | |
| "description": ( | |
| "Opens any application on the computer. " | |
| "Use this whenever the user asks to open, launch, or start any app, " | |
| "website, or program. Always call this tool — never just say you opened it." | |
| ), | |
| "parameters": { | |
| "type": "OBJECT", | |
| "properties": { | |
| "app_name": { | |
| "type": "STRING", | |
| "description": "Exact name of the application (e.g. 'WhatsApp', 'Chrome', 'Spotify')" | |
| } | |
| }, | |
| "required": ["app_name"] | |
| } | |
| }, | |
| { | |
| "name": "web_search", | |
| "description": ( | |
| "Searches the web. Use for ANY question about current facts, events, prices, " | |
| "or topics — always prefer this over guessing. " | |
| "Modes: 'search' (default), 'news' (latest headlines on a topic), " | |
| "'research' (deep comprehensive answer), 'price' (product cost lookup), " | |
| "'compare' (side-by-side comparison of items)." | |
| ), | |
| "parameters": { | |
| "type": "OBJECT", | |
| "properties": { | |
| "query": {"type": "STRING", "description": "Search query or topic"}, | |
| "mode": {"type": "STRING", "description": "search | news | research | price | compare"}, | |
| "items": {"type": "ARRAY", "items": {"type": "STRING"}, "description": "Items to compare (compare mode)"}, | |
| "aspect": {"type": "STRING", "description": "Comparison aspect: price | specs | reviews | features"}, | |
| }, | |
| "required": ["query"] | |
| } | |
| }, | |
| { | |
| "name": "system_monitor_skill", | |
| "description": ( | |
| "Returns real-time system metrics: CPU usage, RAM, GPU load, CPU temperature, " | |
| "uptime, and process count. Use when the user asks about computer performance, " | |
| "temperature, memory, or resource usage." | |
| ), | |
| "parameters": { | |
| "type": "OBJECT", | |
| "properties": {}, | |
| } | |
| }, | |
| { | |
| "name": "weather_report", | |
| "description": "Gives the weather report to user", | |
| "parameters": { | |
| "type": "OBJECT", | |
| "properties": { | |
| "city": {"type": "STRING", "description": "City name"} | |
| }, | |
| "required": ["city"] | |
| } | |
| }, | |
| { | |
| "name": "send_message", | |
| "description": "Sends a text message via WhatsApp, Telegram, or other messaging platform.", | |
| "parameters": { | |
| "type": "OBJECT", | |
| "properties": { | |
| "receiver": {"type": "STRING", "description": "Recipient contact name"}, | |
| "message_text": {"type": "STRING", "description": "The message to send"}, | |
| "platform": {"type": "STRING", "description": "Platform: WhatsApp, Telegram, etc."} | |
| }, | |
| "required": ["receiver", "message_text", "platform"] | |
| } | |
| }, | |
| { | |
| "name": "reminder", | |
| "description": "Sets a timed reminder using Task Scheduler.", | |
| "parameters": { | |
| "type": "OBJECT", | |
| "properties": { | |
| "date": {"type": "STRING", "description": "Date in YYYY-MM-DD format"}, | |
| "time": {"type": "STRING", "description": "Time in HH:MM format (24h)"}, | |
| "message": {"type": "STRING", "description": "Reminder message text"} | |
| }, | |
| "required": ["date", "time", "message"] | |
| } | |
| }, | |
| { | |
| "name": "youtube_video", | |
| "description": ( | |
| "Controls YouTube. Use for: playing videos, summarizing a video's content, " | |
| "getting video info, or showing trending videos." | |
| ), | |
| "parameters": { | |
| "type": "OBJECT", | |
| "properties": { | |
| "action": {"type": "STRING", "description": "play | summarize | get_info | trending (default: play)"}, | |
| "query": {"type": "STRING", "description": "Search query for play action"}, | |
| "save": {"type": "BOOLEAN", "description": "Save summary to Notepad (summarize only)"}, | |
| "region": {"type": "STRING", "description": "Country code for trending e.g. TR, US"}, | |
| "url": {"type": "STRING", "description": "Video URL for get_info action"}, | |
| }, | |
| "required": [] | |
| } | |
| }, | |
| { | |
| "name": "screen_processor", | |
| "description": ( | |
| "Captures the screen or webcam image and lets you analyze it. " | |
| "MUST be called when user asks what is on screen, what you see, " | |
| "look at camera, analyze my screen, etc. " | |
| "You have NO visual ability without this tool. " | |
| "After the image is captured it is sent directly to you — describe what you see and answer the user's question. " | |
| "When using camera: the live view stays open until user says close it or calls close_camera." | |
| ), | |
| "parameters": { | |
| "type": "OBJECT", | |
| "properties": { | |
| "angle": {"type": "STRING", "description": "'screen' to capture display, 'camera' for webcam. Default: 'screen'"}, | |
| "text": {"type": "STRING", "description": "The question or instruction about the captured image"} | |
| }, | |
| "required": ["text"] | |
| } | |
| }, | |
| { | |
| "name": "close_camera", | |
| "description": ( | |
| "Closes the live camera view shown on screen. " | |
| "Call when user says: close camera, stop camera, turn off camera, " | |
| "kamerayı kapat, kapat, creepy, etc." | |
| ), | |
| "parameters": {"type": "OBJECT", "properties": {}, "required": []} | |
| }, | |
| { | |
| "name": "computer_settings", | |
| "description": ( | |
| "Controls the computer: volume, brightness, window management, keyboard shortcuts, " | |
| "typing text on screen, closing apps, fullscreen, dark mode, WiFi, restart, shutdown, " | |
| "scrolling, tab management, zoom, screenshots, lock screen, refresh/reload page. " | |
| "Use for ANY single computer control command." | |
| ), | |
| "parameters": { | |
| "type": "OBJECT", | |
| "properties": { | |
| "action": {"type": "STRING", "description": "The action to perform"}, | |
| "description": {"type": "STRING", "description": "Natural language description of what to do"}, | |
| "value": {"type": "STRING", "description": "Optional value: volume level, text to type, etc."} | |
| }, | |
| "required": [] | |
| } | |
| }, | |
| { | |
| "name": "browser_control", | |
| "description": ( | |
| "Controls any web browser. Use for: opening websites, searching the web, " | |
| "clicking elements, filling forms, scrolling, screenshots, navigation, any web-based task. " | |
| "Always pass the 'browser' parameter when the user specifies a browser (e.g. 'open in Edge', " | |
| "'use Firefox', 'open Chrome'). Multiple browsers can run simultaneously." | |
| ), | |
| "parameters": { | |
| "type": "OBJECT", | |
| "properties": { | |
| "action": {"type": "STRING", "description": "go_to | search | click | type | scroll | fill_form | smart_click | smart_type | get_text | get_url | press | new_tab | close_tab | screenshot | back | forward | reload | switch | list_browsers | close | close_all"}, | |
| "browser": {"type": "STRING", "description": "Target browser: chrome | edge | firefox | opera | operagx | brave | vivaldi | safari. Omit to use the currently active browser."}, | |
| "url": {"type": "STRING", "description": "URL for go_to / new_tab action"}, | |
| "query": {"type": "STRING", "description": "Search query for search action"}, | |
| "engine": {"type": "STRING", "description": "Search engine: google | bing | duckduckgo | yandex (default: google)"}, | |
| "selector": {"type": "STRING", "description": "CSS selector for click/type"}, | |
| "text": {"type": "STRING", "description": "Text to click or type"}, | |
| "description": {"type": "STRING", "description": "Element description for smart_click/smart_type"}, | |
| "direction": {"type": "STRING", "description": "up | down for scroll"}, | |
| "amount": {"type": "INTEGER", "description": "Scroll amount in pixels (default: 500)"}, | |
| "key": {"type": "STRING", "description": "Key name for press action (e.g. Enter, Escape, F5)"}, | |
| "path": {"type": "STRING", "description": "Save path for screenshot"}, | |
| "incognito": {"type": "BOOLEAN", "description": "Open in private/incognito mode"}, | |
| "clear_first": {"type": "BOOLEAN", "description": "Clear field before typing (default: true)"}, | |
| }, | |
| "required": ["action"] | |
| } | |
| }, | |
| { | |
| "name": "file_controller", | |
| "description": "Manages files and folders: list, create, delete, move, copy, rename, read, write, find, disk usage.", | |
| "parameters": { | |
| "type": "OBJECT", | |
| "properties": { | |
| "action": {"type": "STRING", "description": "list | create_file | create_folder | delete | move | copy | rename | read | write | find | largest | disk_usage | organize_desktop | info"}, | |
| "path": {"type": "STRING", "description": "File/folder path or shortcut: desktop, downloads, documents, home"}, | |
| "destination": {"type": "STRING", "description": "Destination path for move/copy"}, | |
| "new_name": {"type": "STRING", "description": "New name for rename"}, | |
| "content": {"type": "STRING", "description": "Content for create_file/write"}, | |
| "name": {"type": "STRING", "description": "File name to search for"}, | |
| "extension": {"type": "STRING", "description": "File extension to search (e.g. .pdf)"}, | |
| "count": {"type": "INTEGER", "description": "Number of results for largest"}, | |
| }, | |
| "required": ["action"] | |
| } | |
| }, | |
| { | |
| "name": "desktop_control", | |
| "description": "Controls the desktop: wallpaper, organize, clean, list, stats.", | |
| "parameters": { | |
| "type": "OBJECT", | |
| "properties": { | |
| "action": {"type": "STRING", "description": "wallpaper | wallpaper_url | organize | clean | list | stats | task"}, | |
| "path": {"type": "STRING", "description": "Image path for wallpaper"}, | |
| "url": {"type": "STRING", "description": "Image URL for wallpaper_url"}, | |
| "mode": {"type": "STRING", "description": "by_type or by_date for organize"}, | |
| "task": {"type": "STRING", "description": "Natural language desktop task"}, | |
| }, | |
| "required": ["action"] | |
| } | |
| }, | |
| { | |
| "name": "code_helper", | |
| "description": "Writes, edits, explains, runs, or builds code files.", | |
| "parameters": { | |
| "type": "OBJECT", | |
| "properties": { | |
| "action": {"type": "STRING", "description": "write | edit | explain | run | build | auto (default: auto)"}, | |
| "description": {"type": "STRING", "description": "What the code should do or what change to make"}, | |
| "language": {"type": "STRING", "description": "Programming language (default: python)"}, | |
| "output_path": {"type": "STRING", "description": "Where to save the file"}, | |
| "file_path": {"type": "STRING", "description": "Path to existing file for edit/explain/run/build"}, | |
| "code": {"type": "STRING", "description": "Raw code string for explain"}, | |
| "args": {"type": "STRING", "description": "CLI arguments for run/build"}, | |
| "timeout": {"type": "INTEGER", "description": "Execution timeout in seconds (default: 30)"}, | |
| }, | |
| "required": ["action"] | |
| } | |
| }, | |
| { | |
| "name": "dev_agent_skill", | |
| "description": ( | |
| "Construit des projets multi-fichiers — petits ou MEGA (launcher, plateforme, SaaS). " | |
| "Détecte automatiquement l'échelle : architecture multi-domaines, phases, marketplace, " | |
| "auth, instances, API, UI, etc. " | |
| "Pour un gros projet : mode analyze d'abord (plan complet), puis execute." | |
| ), | |
| "parameters": { | |
| "type": "OBJECT", | |
| "properties": { | |
| "description": {"type": "STRING", "description": "What the project should do — décrivez TOUTES les fonctionnalités"}, | |
| "language": {"type": "STRING", "description": "Programming language (default: python)"}, | |
| "project_name": {"type": "STRING", "description": "Optional project folder name"}, | |
| "mode": {"type": "STRING", "description": "analyze (plan architecture) | execute (build complet)"}, | |
| "timeout": {"type": "INTEGER", "description": "Run timeout in seconds (default: 30)"}, | |
| }, | |
| "required": ["description"] | |
| } | |
| }, | |
| { | |
| "name": "fallback_agent", | |
| "description": ( | |
| "Routeur intelligent local : Emo décide automatiquement s'il faut exécuter une skill, " | |
| "en créer une nouvelle, ou simplement répondre. " | |
| "Passez la demande telle quelle — pas besoin de préciser « skill » ou le nom de l'outil. " | |
| "La création de skill s'exécute en arrière-plan." | |
| ), | |
| "parameters": { | |
| "type": "OBJECT", | |
| "properties": { | |
| "task": {"type": "STRING", "description": "The developer or skill request to solve."}, | |
| "context": {"type": "STRING", "description": "Optional context, error output, or file list."}, | |
| "folder": {"type": "STRING", "description": "Project folder path (Agent Mode)."}, | |
| }, | |
| "required": ["task"] | |
| } | |
| }, | |
| { | |
| "name": "computer_control", | |
| "description": "Direct computer control: type, click, hotkeys, scroll, move mouse, screenshots, find elements on screen.", | |
| "parameters": { | |
| "type": "OBJECT", | |
| "properties": { | |
| "action": {"type": "STRING", "description": "type | smart_type | click | double_click | right_click | hotkey | press | scroll | move | copy | paste | screenshot | wait | clear_field | focus_window | screen_find | screen_click | random_data | user_data"}, | |
| "text": {"type": "STRING", "description": "Text to type or paste"}, | |
| "x": {"type": "INTEGER", "description": "X coordinate"}, | |
| "y": {"type": "INTEGER", "description": "Y coordinate"}, | |
| "keys": {"type": "STRING", "description": "Key combination e.g. 'ctrl+c'"}, | |
| "key": {"type": "STRING", "description": "Single key e.g. 'enter'"}, | |
| "direction": {"type": "STRING", "description": "up | down | left | right"}, | |
| "amount": {"type": "INTEGER", "description": "Scroll amount (default: 3)"}, | |
| "seconds": {"type": "NUMBER", "description": "Seconds to wait"}, | |
| "title": {"type": "STRING", "description": "Window title for focus_window"}, | |
| "description": {"type": "STRING", "description": "Element description for screen_find/screen_click"}, | |
| "type": {"type": "STRING", "description": "Data type for random_data"}, | |
| "field": {"type": "STRING", "description": "Field for user_data: name|email|city"}, | |
| "clear_first": {"type": "BOOLEAN", "description": "Clear field before typing (default: true)"}, | |
| "path": {"type": "STRING", "description": "Save path for screenshot"}, | |
| }, | |
| "required": ["action"] | |
| } | |
| }, | |
| { | |
| "name": "game_updater", | |
| "description": ( | |
| "THE ONLY tool for ANY Steam or Epic Games request. " | |
| "Use for: installing, downloading, updating games, listing installed games, " | |
| "checking download status, scheduling updates. " | |
| "ALWAYS call directly for any Steam/Epic/game request. " | |
| "NEVER use browser_control or web_search for Steam/Epic." | |
| ), | |
| "parameters": { | |
| "type": "OBJECT", | |
| "properties": { | |
| "action": {"type": "STRING", "description": "update | install | list | download_status | schedule | cancel_schedule | schedule_status (default: update)"}, | |
| "platform": {"type": "STRING", "description": "steam | epic | both (default: both)"}, | |
| "game_name": {"type": "STRING", "description": "Game name (partial match supported)"}, | |
| "app_id": {"type": "STRING", "description": "Steam AppID for install (optional)"}, | |
| "hour": {"type": "INTEGER", "description": "Hour for scheduled update 0-23 (default: 3)"}, | |
| "minute": {"type": "INTEGER", "description": "Minute for scheduled update 0-59 (default: 0)"}, | |
| "shutdown_when_done": {"type": "BOOLEAN", "description": "Shut down PC when download finishes"}, | |
| }, | |
| "required": [] | |
| } | |
| }, | |
| { | |
| "name": "flight_finder", | |
| "description": "Searches Google Flights and speaks the best options.", | |
| "parameters": { | |
| "type": "OBJECT", | |
| "properties": { | |
| "origin": {"type": "STRING", "description": "Departure city or airport code"}, | |
| "destination": {"type": "STRING", "description": "Arrival city or airport code"}, | |
| "date": {"type": "STRING", "description": "Departure date (any format)"}, | |
| "return_date": {"type": "STRING", "description": "Return date for round trips"}, | |
| "passengers": {"type": "INTEGER", "description": "Number of passengers (default: 1)"}, | |
| "cabin": {"type": "STRING", "description": "economy | premium | business | first"}, | |
| "save": {"type": "BOOLEAN", "description": "Save results to Notepad"}, | |
| }, | |
| "required": ["origin", "destination", "date"] | |
| } | |
| }, | |
| { | |
| "name": "shutdown_jarvis", | |
| "description": ( | |
| "Shuts down the assistant completely. " | |
| "Call this when the user expresses intent to end the conversation, " | |
| "close the assistant, say goodbye, or stop Jarvis. " | |
| "The user can say this in ANY language." | |
| ), | |
| "parameters": { | |
| "type": "OBJECT", | |
| "properties": {}, | |
| } | |
| }, | |
| { | |
| "name": "file_processor", | |
| "description": ( | |
| "Processes any file that the user has uploaded or dropped onto the interface. " | |
| "Use this when the user refers to an uploaded file and wants an action on it. " | |
| "Supports: images (describe/ocr/resize/compress/convert), " | |
| "PDFs (summarize/extract_text/to_word), " | |
| "Word docs & text files (summarize/fix/reformat/translate), " | |
| "CSV/Excel (analyze/stats/filter/sort/convert), " | |
| "JSON/XML (validate/format/analyze), " | |
| "code files (explain/review/fix/optimize/run/document/test), " | |
| "audio (transcribe/trim/convert/info), " | |
| "video (trim/extract_audio/extract_frame/compress/transcribe/info), " | |
| "archives (list/extract), " | |
| "presentations (summarize/extract_text). " | |
| "ALWAYS call this tool when a file has been uploaded and the user gives a command about it. " | |
| "If the user's command is ambiguous, pick the most logical action for that file type." | |
| ), | |
| "parameters": { | |
| "type": "OBJECT", | |
| "properties": { | |
| "file_path": { | |
| "type": "STRING", | |
| "description": "Full path to the uploaded file. Leave empty to use the currently uploaded file." | |
| }, | |
| "action": { | |
| "type": "STRING", | |
| "description": ( | |
| "What to do with the file. Examples by type:\n" | |
| "image: describe | ocr | resize | compress | convert | info\n" | |
| "pdf: summarize | extract_text | to_word | info\n" | |
| "docx/txt: summarize | fix | reformat | translate_hint | word_count | to_bullet\n" | |
| "csv/excel: analyze | stats | filter | sort | convert | info\n" | |
| "json: validate | format | analyze | to_csv\n" | |
| "code: explain | review | fix | optimize | run | document | test\n" | |
| "audio: transcribe | trim | convert | info\n" | |
| "video: trim | extract_audio | extract_frame | compress | transcribe | info | convert\n" | |
| "archive: list | extract\n" | |
| "pptx: summarize | extract_text | analyze" | |
| ) | |
| }, | |
| "instruction": { | |
| "type": "STRING", | |
| "description": "Free-form instruction if action doesn't cover it. E.g. 'translate this to Turkish', 'find all email addresses'" | |
| }, | |
| "format": { | |
| "type": "STRING", | |
| "description": "Target format for conversion. E.g. 'mp3', 'pdf', 'csv', 'png'" | |
| }, | |
| "width": {"type": "INTEGER", "description": "Target width for image resize"}, | |
| "height": {"type": "INTEGER", "description": "Target height for image resize"}, | |
| "scale": {"type": "NUMBER", "description": "Scale factor for image resize (e.g. 0.5)"}, | |
| "quality": {"type": "INTEGER", "description": "Quality 1-100 for image/video compress"}, | |
| "start": {"type": "STRING", "description": "Start time for trim: seconds or HH:MM:SS"}, | |
| "end": {"type": "STRING", "description": "End time for trim: seconds or HH:MM:SS"}, | |
| "timestamp": {"type": "STRING", "description": "Timestamp for video frame extraction HH:MM:SS"}, | |
| "column": {"type": "STRING", "description": "Column name for CSV filter/sort"}, | |
| "value": {"type": "STRING", "description": "Filter value for CSV filter"}, | |
| "condition": {"type": "STRING", "description": "Filter condition: equals|contains|gt|lt"}, | |
| "ascending": {"type": "BOOLEAN", "description": "Sort order for CSV sort (default: true)"}, | |
| "save": {"type": "BOOLEAN", "description": "Save result to file (default: true)"}, | |
| "destination": {"type": "STRING", "description": "Output folder for archive extract"}, | |
| }, | |
| "required": [] | |
| } | |
| }, | |
| { | |
| "name": "save_memory", | |
| "description": ( | |
| "Save an important personal fact about the user to long-term memory. " | |
| "Call this silently whenever the user reveals something worth remembering: " | |
| "name, age, city, job, preferences, hobbies, relationships, projects, or future plans. " | |
| "Do NOT call for: weather, reminders, searches, or one-time commands. " | |
| "Do NOT announce that you are saving — just call it silently. " | |
| "Values must be in English regardless of the conversation language." | |
| ), | |
| "parameters": { | |
| "type": "OBJECT", | |
| "properties": { | |
| "category": { | |
| "type": "STRING", | |
| "description": ( | |
| "identity — name, age, birthday, city, job, language, nationality | " | |
| "preferences — favorite food/color/music/film/game/sport, hobbies | " | |
| "projects — active projects, goals, things being built | " | |
| "relationships — friends, family, partner, colleagues | " | |
| "wishes — future plans, things to buy, travel dreams | " | |
| "notes — habits, schedule, anything else worth remembering" | |
| ) | |
| }, | |
| "key": {"type": "STRING", "description": "Short snake_case key (e.g. name, favorite_food, sister_name)"}, | |
| "value": {"type": "STRING", "description": "Concise value in English (e.g. Fatih, pizza, older sister)"}, | |
| }, | |
| "required": ["category", "key", "value"] | |
| } | |
| }, | |
| ] | |
| def get_tool_declarations() -> list[dict]: | |
| decls = list(_TOOL_DECLARATIONS) | |
| decls.append(skill_tool_declaration()) | |
| return decls | |