diff --git a/docs/res/usage/webui/project-active-chat.png b/docs/res/usage/webui/project-active-chat.png index 91b3f27ffe80c8cce0f052cdcfe797bbaee603bc..e6bba3e839f4a6edcb3add21f049c0a00140bbdd 100644 Binary files a/docs/res/usage/webui/project-active-chat.png and b/docs/res/usage/webui/project-active-chat.png differ diff --git a/docs/res/usage/webui/project-create-filled.png b/docs/res/usage/webui/project-create-filled.png index 4e927401205235983bcaa3cbef82ba615689a390..a6848bea9c0089bb4cdd1681a39457cedea13fee 100644 Binary files a/docs/res/usage/webui/project-create-filled.png and b/docs/res/usage/webui/project-create-filled.png differ diff --git a/docs/res/usage/webui/project-instructions-filled.png b/docs/res/usage/webui/project-instructions-filled.png index 5a6c035e5e8f5b22d6be6fca2b3de9587d733049..3a0677284a4f47510232d70946f5aef2538a280a 100644 Binary files a/docs/res/usage/webui/project-instructions-filled.png and b/docs/res/usage/webui/project-instructions-filled.png differ diff --git a/docs/res/usage/webui/project-picker.png b/docs/res/usage/webui/project-picker.png index 969799eef577e548dd9d0a1fa6b2a6663dee2845..cea753bfeda7e1cd3e40003f688582be4fab418e 100644 Binary files a/docs/res/usage/webui/project-picker.png and b/docs/res/usage/webui/project-picker.png differ diff --git a/docs/res/usage/webui/projects-empty.png b/docs/res/usage/webui/projects-empty.png index 57546dee9c5d6fb81ef2eecbde7e9d632afed42d..94ab70c26ffcb72b67ecd1cfe2f7e77e37464f9f 100644 Binary files a/docs/res/usage/webui/projects-empty.png and b/docs/res/usage/webui/projects-empty.png differ diff --git a/docs/res/usage/webui/projects-list-created.png b/docs/res/usage/webui/projects-list-created.png index 4ea2009741c14340d18751f24e0244ae8894a463..f75e64bfde61d20666a8eabcf43878e19f911c31 100644 Binary files a/docs/res/usage/webui/projects-list-created.png and b/docs/res/usage/webui/projects-list-created.png differ diff --git a/docs/res/usage/webui/skills-selector-checked.png b/docs/res/usage/webui/skills-selector-checked.png index b19aa9c29b9692d02717614ab099026015842d1c..fb30458d2a369a2fcdd0240032a9e783b6be14de 100644 Binary files a/docs/res/usage/webui/skills-selector-checked.png and b/docs/res/usage/webui/skills-selector-checked.png differ diff --git a/docs/res/usage/webui/skills-selector.png b/docs/res/usage/webui/skills-selector.png index 5bbfa73f17731262a0f589df75c01af37a016c37..5063638685361fee94459c895ba544691bd020a0 100644 Binary files a/docs/res/usage/webui/skills-selector.png and b/docs/res/usage/webui/skills-selector.png differ diff --git a/docs/res/usage/webui/unread-dot-chat-list.png b/docs/res/usage/webui/unread-dot-chat-list.png index 0d089aba26c1b9587b6302a2c00acf43d8527e11..1b0622cde689a9a01ab3d3c359d0b5d6d9a984b7 100644 Binary files a/docs/res/usage/webui/unread-dot-chat-list.png and b/docs/res/usage/webui/unread-dot-chat-list.png differ diff --git a/docs/setup/dev-setup.md b/docs/setup/dev-setup.md index 9cec423d7ca9190c3c86d619efc869f0d70e16b3..d1390d0de75c17ad56bae225b2b852f12d050943 100644 --- a/docs/setup/dev-setup.md +++ b/docs/setup/dev-setup.md @@ -1,3 +1,190 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:71d5be20f1035b01e37236efe2d1baedaf829baf99e575cf841b9db2d47f96b6 -size 12391 +# Development manual for Agent Zero +This guide will show you how to setup a local development environment for Agent Zero in a VS Code compatible IDE, including proper debugger. + + +[![Tutorial video](../res/devguide_vid.png)](https://www.youtube.com/watch?v=KE39P4qBjDk) + + + +> [!WARNING] +> This guide is for developers and contributors. It assumes you have a basic understanding of how to use Git/GitHub, Docker, IDEs and Python. + +> [!NOTE] +> - Agent Zero runs in a Docker container, this simplifies installation and ensures unified environment and behavior across systems. +> - Developing and debugging in a container would be complicated though, therefore we use a hybrid approach where the python framework runs on your machine (in VS Code for example) and only connects to a Dockerized instance when it needs to execute code or use other pre-installed functionality like the built-in search engine. + + +## To follow this guide you will need: +1. VS Code compatible IDE (VS Code, Cursor, Windsurf...) +2. Python environment (Conda, venv, uv...) +3. Docker (Docker Desktop, docker-ce...) +4. (optional) Git/GitHub + +> [!NOTE] +> I will be using clean VS Code, Conda and Docker Desktop in this example on MacOS. + + +## Step 0: Install required software +- See the list above and install the software required if you don't already have it. +- You can choose your own variants, but Python, Docker and a VS Code compatible IDE are required. +- For Python you can choose your environment manager - base Python venv, Conda, uv... + +## Step 1: Clone or download the repository +- Agent Zero is available on GitHub [github.com/agent0ai/agent-zero](https://github.com/agent0ai/agent-zero). +- You can download the files using a browser and extract or run `git clone https://github.com/agent0ai/agent-zero` in your desired directory. + +> [!NOTE] +> In my case, I used `cd ~/Desktop` and `git clone https://github.com/agent0ai/agent-zero`, so my project folder is `~/Desktop/agent-zero`. + +## Step 2: Open project folder in your IDE +- I will be using plain and clean VS Code for this example to make sure I don't skip any setup part, you can use any of it's variants like Cursor, Windsurf etc. +- Agent Zero comes with `.vscode` folder that contains basic setup, recommended extensions, and debugger profiles. These will help us a lot. + +1. Open your IDE and open the project folder using `File > Open Folder` and select your folder, in my case `~/Desktop/agent-zero`. +2. You will probably be prompted to trust the directory, confirm that. +3. You should now have the project open in your IDE +![VS Code project](../res/dev/devinst-1.png) + +# Step 3: Prepare your IDE: +1. Notice the prompt in lower right corner of the screenshot above to install recommended extensions, this comes from the `.vscode/extensions.json` file. It contains Python language support, debugger and error helper, install them by confirming the popup or manually in Extensions tab of your IDE. These are the extensions mentioned: +``` +usernamehw.errorlens +ms-python.debugpy +ms-python.python +``` + +Now when you select one of the python files in the project, you should see proper Python syntax highlighting and error detection. It should immediately show some errors, because we did not yet install dependencies. +![VS Code Python](../res/dev/devinst-2.png) + +2. Prepare the python environment to run Agent Zero in. This step assumes you have some Python runtime installed. By clicking the python version in lower right corner (3.13.1 in my example), you should get a list of available environments. You can click the `+ Create Virtual Environment` button. You might be prompted to select the environment manager if you have multiple installed. I have venv and Conda, I will select Conda here. I'm also prompted for desired python version, I will select 3.12, that is known to work well. +![VS Code Python environments](../res/dev/devinst-3.png) +![VS Code Python environments](../res/dev/devinst-4.png) + +- Your new environment should be automatically activated. If not, select it in the lower right corner. You might need to open a new terminal in VS Code to reflect the changes with `Terminal > New Terminal` or clicking the `+` button in the terminal tab. Your terminal prompt should now start with your environment name/path, in my case `(/Users/frdel/Desktop/agent-zero/.conda)` This shows the environment is active in the terminal. + +![VS Code env terminal](../res/dev/devinst-5.png) + +3. Install dependencies. Run these commands from the project root: + +```bash +pip install -r requirements.txt +PLAYWRIGHT_BROWSERS_PATH=./tmp/playwright patchright install chromium --no-shell +``` + +The first command installs Python dependencies. + +The second command installs full Patchright Chromium into `./tmp/playwright`, +relative to the project root. Docker images use the absolute path +`/a0/tmp/playwright` and ship Chromium preinstalled. + +If you skip the second command, local development can still download Chromium on +first Browser use through `ensure_playwright_binary()` in +`plugins/_browser/helpers/playwright.py`. Pre-installing avoids that wait. + +See the [Browser Guide](../guides/browser.md) for the Browser surface, +screenshots, annotations, and host-browser mode. +Errors in the code editor caused by missing packages should now be gone. If not, try reloading the window. + + +## Step 4: Run Agent Zero in the IDE +Great work! Now you should be able to run Agent Zero from your IDE including real-time debugging. +It will not be able to do code execution and few other features requiring the Docker container just yet, but most of the framework will already work. + +1. The project is pre-configured for debugging. Go to Debugging tab, select "run_ui.py" and click the green play button (or press F5 by default). The configuration can be found at `.vscode/launch.json`. + +![VS Code debugging](../res/dev/devinst-6.png) + +The framework will run at the default port 5000. If you open `http://localhost:5000` in your browser and see `ERR_EMPTY_RESPONSE`, don't panic, you may need to select another port like I did for some reason. If you need to change the default port, you can add `"--port=5555"` to the args in the `.vscode/launch.json` file or you can create a `.env` file in the root directory and set the `WEB_UI_PORT` variable to the desired port. + +You can also set the bind host via `"--host=0.0.0.0"` (or `WEB_UI_HOST=0.0.0.0`). + +It may take a while the first time. You should see output like the screenshot below. The RFC error is ok for now as we did not yet connect our local development to another instance in docker. +![First run](../res/dev/devinst-7.png) + + +After inserting my API key in settings, my Agent Zero instance works. I can send a simple message and get a response. +Some tools like code execution will not work yet because they need to be connected to a Dockerized instance. + +![First message](../res/dev/devinst-8.png) + + +## Debugging +- You can try out the debugger already by placing a breakpoint somewhere in the python code. +- Let's open `python/api/message.py` for example and place a breakpoint at the beginning of the `communicate` function by clicking on the left of the row number. A red dot should appear showing a breakpoint is set. + +![Debugging](../res/dev/devinst-9.png) + +- Now when I send a message in the UI, the debugger will pause the execution at the breakpoint and allow me to inspect all the runtime variables and run the code step by step, even modify the variables or jump to another locations in the code. No more print statements needed! + +![Debugging](../res/dev/devinst-10.png) + + +## Step 5: Run another instance of Agent Zero in Docker +- Some parts of A0 require standardized linux environment, additional web services and preinstalled binaries that would be unneccessarily complex to set up in a local environment. +- To make development easier, we can use existing A0 instance in docker and forward some requests to be executed there using SSH and RFC (Remote Function Call). + +1. Pull the docker image `agent0ai/agent-zero` from Docker Hub and run it with a web port (`80`) mapped and SSH port (`22`) mapped. +If you want, you can also map the `/a0` folder to our local project folder as well, this way we can update our local instance and the docker instance at the same time. +This is how it looks in my example: port `80` is mapped to `8880` on the host and `22` to `8822`, `/a0` folder mapped to `/Users/frdel/Desktop/agent-zero`: + +![docker run](../res/dev/devinst-11.png) +![docker run](../res/dev/devinst-12.png) + + +## Step 6: Configure SSH and RFC connection +- The last step is to configure the local development (VS Code) instance and the dockerized instance to communicate with each other. This is very simple and can be done in the settings in the Web UI of both instances. +- In my example the dark themed instance is the VS Code one, the light themed one is the dockerized instance. + +1. Open the "Settings" page in the Web UI of your dockerized instance and go in the "Development" section. +2. Set the `RFC Password` field to a new password and save. +3. Open the "Settings" page in the Web UI of your local instance and go in the "Development" section. +4. Here set the `RFC Password` field to the same password you used in the dockerized instance. Also set the SSH port and HTTP port the same numbers you used when creating the container - in my case `8822` for SSH and `8880` for HTTP. The `RFC Destination URL` will most probably stay `localhost` as both instances are running on the host machine. +5. Click save and test by asking your agent to do something in the terminal, like "Get current OS version". It should be able to communicate with the dockerized instance via RFC and SSH and execute the command there, responding with something like "Kali GNU/Linux Rolling". + +My Dockerized instance: +![Dockerized instance](../res/dev/devinst-14.png) + +My VS Code instance: +![VS Code instance](../res/dev/devinst-13.png) + +## RFC Notes (Host IDE + Docker Execution) +Agent Zero runs code inside the container by default. If you are running the framework locally in your IDE but want tools (like code execution) to run in Docker, configure RFC in **Settings -> Development** and point it to a running Agent Zero container. This routes execution through SSH/RFC to the container while keeping the UI and agent loop on your host. + + +# Congratulations! + +You have successfully set up a complete Agent Zero development environment! You now have: + +- A local development instance running in your IDE with full debugging capabilities +- A dockerized instance for code execution and system operations +- RFC and SSH communication between both instances +- The ability to develop, debug, and test Agent Zero features seamlessly + +You're now ready to contribute to Agent Zero, create custom extensions, or modify the framework to suit your needs. Happy coding! + + +## Next steps +- See [Create a Small Plugin](../guides/create-plugin.md) before building a new plugin. +- Use [DeepWiki for Agent Zero](https://deepwiki.com/agent0ai/agent-zero) for architecture and source-linked internals. +- See [Contributing to Agent Zero](../guides/contribution.md) for contribution basics. + +## Configuration via Environment Variables + +For development and testing, you can override default settings using the `.env` file with `A0_SET_` prefixed variables: + +```env +# Add to your .env file +A0_SET_chat_model_provider=ollama +A0_SET_chat_model_name=llama3.2 +A0_SET_chat_model_api_base=http://localhost:11434 +A0_SET_memory_recall_interval=5 +``` + +These environment variables automatically override the hardcoded defaults in `get_default_settings()` without modifying code. Useful for testing different configurations or multi-environment setups. + +## Want to build your docker image? +- You can use the `DockerfileLocal` to build your docker image. +- Navigate to your project root in the terminal and run `docker build -f DockerfileLocal -t agent-zero-local --build-arg CACHE_DATE=$(date +%Y-%m-%d:%H:%M:%S) .` +- The `CACHE_DATE` argument is optional, it is used to cache most of the build process and only rebuild the last steps when the files or dependencies change. +- See `docker/run/build.txt` for more build command examples. +- Automated Docker Hub publishing for release tags is handled by `.github/workflows/docker-publish.yml`. The latest eligible `main` tag generates its GitHub release body on the fly from commit subjects and descriptions via OpenRouter. diff --git a/docs/setup/installation.md b/docs/setup/installation.md index dfe606e29dc6f6148828ace04e5d3ad0d24fb5cf..b0e826750e4d8143bb7b555a08164d23ecbd0f44 100644 --- a/docs/setup/installation.md +++ b/docs/setup/installation.md @@ -1,3 +1,871 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:22679137b4317a0dafc352dca421aa13cd208db65b9b9deb843c6198730cfd6f -size 35606 +# Installation Guide + +## **Goal:** Go from zero to a first working chat with minimal setup. + +--- + +## Quick Start (Recommended) + +Agent Zero runs as a Docker container, and you now have two friendly ways to get +there: + +- **A0 Launcher** is the desktop app. It can download Agent Zero, create and + manage Instances, and help set up the local container runtime when needed. +- **A0 Install** is the terminal installer. It is best for SSH sessions, + servers, scripted setup, recovery shells, or users who prefer commands. + +If Docker is already installed and running, you can also start the container +directly. + +### A0 Launcher + +Use **A0 Launcher** when you want the guided desktop path. Download the app for +your platform, open it, and let it check Docker or set up a runtime before it +downloads Agent Zero. + +#### Downloads + +| Architecture | macOS | Linux | Windows | +| --- | --- | --- | --- | +| x86 | [Mac Intel](https://github.com/agent0ai/a0-launcher/releases/download/v0.9/a0-launcher-0.9-macos-x64.dmg) | [Linux x86](https://github.com/agent0ai/a0-launcher/releases/download/v0.9/a0-launcher-0.9-linux-x64.AppImage) | [Windows x86](https://github.com/agent0ai/a0-launcher/releases/download/v0.9/a0-launcher-0.9-windows-x64.exe) | +| ARM64 | [Mac Apple Silicon](https://github.com/agent0ai/a0-launcher/releases/download/v0.9/a0-launcher-0.9-macos-arm64.dmg) | [Linux ARM64](https://github.com/agent0ai/a0-launcher/releases/download/v0.9/a0-launcher-0.9-linux-arm64.AppImage) | [Windows ARM64](https://github.com/agent0ai/a0-launcher/releases/download/v0.9/a0-launcher-0.9-windows-arm64.exe) | + +See the [A0 Launcher v0.9 release](https://github.com/agent0ai/a0-launcher/releases/tag/v0.9) +for release notes and updater metadata. See the +[Launcher guide](../guides/launcher.md) for the first-run walkthrough. + +### A0 Install + +Use **A0 Install** when you want the command-line path. The installer creates a +Dockerized Agent Zero instance, mounts user data to `/a0/usr`, and tries to +reuse an existing Docker-compatible runtime before setting one up. + +#### macOS / Linux +```bash +curl -fsSL https://bash.agent-zero.ai | bash +``` + +#### Windows PowerShell +```powershell +irm https://ps.agent-zero.ai | iex +``` + +#### Headless / scripted + +For servers and automation, Quick Start mode creates one instance and exits +without opening menus: + +```bash +curl -fsSL https://bash.agent-zero.ai | bash -s -- --quick-start --name agent-zero --port 5080 +``` + +```powershell +& ([scriptblock]::Create((irm https://ps.agent-zero.ai))) -QuickStart -Name agent-zero -Port 5080 +``` + +Use `--skip-runtime-setup` / `-SkipRuntimeSetup` when Docker must already be +working and the installer should not try to set up a runtime. See the +[A0 Install repository](https://github.com/agent0ai/a0-install) for all +installer flags. + +### Docker already installed? Run this directly + +```bash +docker run -p 80:80 -v a0_usr:/a0/usr agent0ai/agent-zero +``` + +Once the install completes, open the URL shown in your terminal or Launcher to +access the Web UI. Complete onboarding, add your model provider or API key, then +continue to [Step 3: Configure Agent Zero](#step-3-configure-agent-zero). + +> [!TIP] +> Need Agent Zero to reach host-machine files, shell, or a host browser? Install the optional [A0 CLI Connector](../guides/a0-cli-connector.md), then run `a0` to connect your terminal to this Agent Zero instance. + +--- + +## How to Update Agent Zero + +### Self Update (Recommended) + +Use the built-in updater in the Web UI: + +1. Open **Settings UI -> Update** tab +2. Open **Self Update** +3. Wait for the update checker to see if you have the latest version or if there's an available update. + +You'll also be prompted through the UI when a new A0 version is released. Backups are automatically managed internally during this process. + +For technical details of the updater, see [Self Update](../guides/self-update.md). + +### Updating from v1.20 to v2.0 + +Agent Zero v2.0 starts a new major release line. If your instance is on v1.20, +the in-app Self Update can show the newer v2.x line, but it will not apply that +jump inside the existing v1 Docker image. The safe path is: + +1. Create a backup zip from the old v1.20 instance. +2. Pull the new `agent0ai/agent-zero:latest` Docker image. For the v2.0 release, + `latest` is the v2.0 image. +3. Start a new container from that image. +4. Restore the backup zip into the new v2.0 instance. + +![Self Update warning for a newer major release line](../res/usage/updating/self-update-v1-to-v2-warning.png) + +#### Without Agent Zero Launcher + +Use this path if you manage Agent Zero directly from Docker Desktop or Docker +CLI. + +1. Open your old v1.20 Web UI and create a backup from **Settings -> Check for Updates -> Backup & Restore -> Create Backup**. Keep the downloaded `.zip` file. +2. Pull the v2.0 image. In **Docker Desktop**, search for `agent0ai/agent-zero:latest` and pull that image. In **Docker CLI**, run: + ```bash + docker pull agent0ai/agent-zero:latest + ``` +3. Start a new v2.0 container on a different host port so the old instance stays available: + ```bash + docker run -d -p 50081:80 --name agent-zero-v2 -v a0_v2_usr:/a0/usr agent0ai/agent-zero:latest + ``` +4. Open the new v2.0 instance, complete any first-run prompts, then restore the downloaded `.zip` from **Settings -> Check for Updates -> Backup & Restore -> Restore Backup**. +5. Verify chats, projects, memory, settings, and custom plugins before removing the old v1.20 container. + +#### With Agent Zero Launcher + +Launcher gives you the same backup/restore idea from the **Instances** page. + +1. Open **Instances**, choose the old v1.20 Instance, and use **Backup `/a0/usr`**. +2. Open **Installs**, use the **latest** card, then **Install** or **Run** the image. For the v2.0 release, **latest** is the v2.0 image. +3. Return to **Instances**, choose the new v2.0 Instance, and use **Restore `/a0/usr`** with the backup zip. +4. Open the new Instance and verify it before deleting or stopping the old v1.20 container. + +Launcher keeps old and new Instances visible separately, which makes it easier +to compare them before cleanup. + +> [!CAUTION] +> Do not try to solve the v1.20 -> v2.0 jump by bind-mounting the whole old +> `/a0` directory into a new container. Keep user data under `/a0/usr`, use the +> backup/restore flow, and let the new image provide the v2.0 system files. + +### Updating from Pre-v0.9.8 + +If you are upgrading from Agent Zero v0.9.8 or earlier to v1.1 or newer, use the migration path below. Older installs were laid out differently, so the in-app Self Update is not the right tool for that jump. + +1. **Backup your existing `usr/` directory** (which contains your settings, projects, memory, and custom plugins). +2. **Run the new install script** to set up the current Docker-based install: + - macOS / Linux: `curl -fsSL https://bash.agent-zero.ai | bash` + - Windows (PowerShell): `irm https://ps.agent-zero.ai | iex` +3. **Migrate your data:** After the new installation completes, copy the contents of your backed-up `usr/` directory into the new `/a0/usr/` directory created by the script. +4. Restart the container for the changes to take effect. + +### Manual Update (Advanced) + +> Use this only if Self Update is unavailable or you must manage containers yourself (for example, some custom Docker setups). + +1. Keep the current container running +2. `docker pull agent0ai/agent-zero:latest` +3. Start a **new** container on a different host port, for example: `docker run -d -p 50081:80 --name agent-zero-new agent0ai/agent-zero:latest` +4. On the **old** instance: **Settings -> Check for Updates -> Backup & Restore -> Create Backup** +5. On the **new** instance: restore the downloaded backup zip +6. Verify chats and data, then remove the old container + +> [!CAUTION] +> Do not delete the old container until the new one has your data. + +> [!TIP] +> If the new instance fails to load settings, remove `/a0/usr/settings.json` and restart to regenerate default settings. + +--- + +## Manual Installation (Advanced) + +> Users should use [Quick Start (Recommended)](#quick-start-recommended) above. The steps below are for custom Docker setups, air-gapped installs, or when you cannot use the install scripts. + +Follow the steps below to install Docker and run the image by hand. + +### Step 1: Install Docker Desktop + +Docker Desktop provides the runtime environment for Agent Zero, ensuring consistent behavior and security across platforms. The entire framework runs within a Docker container, providing isolation and easy deployment. + +**Choose your operating system:** + + + + + + + +
+ +Windows
+Windows + +		 + +macOS
+macOS + +		 + +Linux
+Linux + +
+ +--- + + +#### Windows Windows Installation + +**1.1. Download Docker Desktop** + +Go to the [Docker Desktop download page](https://www.docker.com/products/docker-desktop/) and download the Windows version (Intel/AMD is the main download button). + +docker download +

+ +**1.2. Run the Installer** + +Run the installer with default settings. + +docker install +docker install +

+ +**1.3. Launch Docker Desktop** + +Once installed, launch Docker Desktop from your Start menu or desktop shortcut. + +docker installed + +**Docker is now installed.** + +Continue to [Step 2: Run Agent Zero](#step-2-run-agent-zero) + +--- + + +#### macOS macOS Installation + +**1.1. Download Docker Desktop** + +Go to the [Docker Desktop download page](https://www.docker.com/products/docker-desktop/) and download the macOS version (choose Apple Silicon or Intel based on your Mac). + +docker download +

+ +**1.2. Install Docker Desktop** + +Drag and drop the Docker application to your Applications folder. + +docker install +

+ +**1.3. Launch Docker Desktop** + +Open Docker Desktop from your Applications folder. + +docker installed +

+ +**1.4. Configure Docker Socket** + +> [!NOTE] +> **Important macOS Configuration:** In Docker Desktop's preferences (Docker menu) -> Settings -> Advanced, enable "Allow the default Docker socket to be used (requires password)." + +![docker socket macOS](../res/setup/macsocket.png) + +**Docker is now installed.** + +Continue to [Step 2: Run Agent Zero](#step-2-run-agent-zero) + +--- + + +#### Linux Linux Installation + +**1.1. Choose Installation Method** + +You can install either Docker Desktop or docker-ce (Community Edition). + +**Option A: Docker Desktop (Recommended for beginners)** + +Follow the instructions for your specific Linux distribution [here](https://docs.docker.com/desktop/install/linux-install/). + +**Option B: docker-ce (Lightweight alternative)** + +Follow the installation instructions [here](https://docs.docker.com/engine/install/). + +**1.2. Post-Installation Steps (docker-ce only)** + +If you installed docker-ce, add your user to the `docker` group: + +```bash +sudo usermod -aG docker $USER +``` + +Log out and back in, then authenticate: + +```bash +docker login +``` + +**1.3. Launch Docker** + +If you installed Docker Desktop, launch it from your applications menu. + +**Docker is now installed.** + +> [!TIP] +> **Deploying on a VPS/Server?** For production deployments with reverse proxy, SSL, and domain configuration, see the [VPS Deployment Guide](vps-deployment.md). + +--- + +### Step 2: Run Agent Zero + +#### 2.1. Pull the Agent Zero Docker Image + +**Using Docker Desktop GUI:** + +- Search for `agent0ai/agent-zero` in Docker Desktop +- Click the `Pull` button +- The image will be downloaded to your machine in a few minutes + +![docker pull](../res/setup/1-docker-image-search.png) + +**Using Terminal:** + +```bash +docker pull agent0ai/agent-zero +``` + +#### 2.2. (Optional) Map Folders for Persistence + +Choose or create a folder on your computer where Agent Zero will save its data. + +Setting up persistence is needed only if you want your data and files to remain available even after you delete the container. + +You can pick any location you find convenient: + +- **Windows:** `C:\agent-zero-data` +- **macOS/Linux:** `/home/user/agent-zero-data` + +You can map just the `/a0/usr` directory (recommended) or individual subfolders of `/a0` to a local directory. + +> [!CAUTION] +> Do **not** map the entire `/a0` directory: it contains the application code and can break upgrades. + +> [!TIP] +> Choose a location that's easy to access and backup. All your Agent Zero data will be directly accessible in this directory. + +#### 2.3. Run the Container + +**Using Docker Desktop GUI:** + +- In Docker Desktop, go to the "Images" tab +- Click the `Run` button next to the `agent0ai/agent-zero` image +- Open the "Optional settings" menu +- **Ensure at least one host port is mapped to container port `80`** (set host port to `0` for automatic assignment) +- Click the `Run` button + +![docker port mapping](../res/setup/2-docker-image-run.png) +![docker volume mapping](../res/setup/2-docker-image-run-3.png) + +The container will start and show in the "Containers" tab: + +![docker containers](../res/setup/4-docker-container-started.png) + +#### 2.4. Access the Web UI + +The framework will take a few seconds to initialize. Find the mapped port in Docker Desktop (shown as `:80`) or click the port right under the container ID: + +![docker logs](../res/setup/5-docker-click-to-open.png) + +Open `http://localhost:` in your browser. The Web UI will open - Agent Zero is ready for configuration! + +![docker ui](../res/setup/6-docker-a0-running-new.png) + +> [!TIP] +> You can also access the Web UI by clicking the port link directly under the container ID in Docker Desktop. + +> [!NOTE] +> After starting the container, you'll find all Agent Zero files in your chosen directory. You can access and edit these files directly on your machine, and the changes will be immediately reflected in the running container. + +**Running A0 using Terminal?** + +```bash +docker run -p 0:80 -v /path/to/your/work_dir:/a0/usr agent0ai/agent-zero +``` + +- Replace `0` with a fixed port if you prefer (e.g., `50080:80`) + +--- + +## Step 3: Configure Agent Zero + +The UI opens on the welcome screen. If model setup is missing, send a message +or use the setup shortcuts to choose Cloud, AI account, or Local access, then +select your main and utility models. For the screenshot walkthrough, see the +[First-Run Onboarding guide](../guides/onboarding.md). + +### Settings Configuration + +Agent Zero provides a comprehensive settings interface to customize various aspects of its functionality. Access the settings by clicking the "Settings" button with a gear icon in the sidebar. + +### Agent Configuration + +- **Agent Profile:** Select the default profile for new chats, such as `agent0`, + `hacker`, or `researcher`. +- **Memory Subdirectory:** Select the subdirectory for agent memory storage, allowing separation between different instances. +- **Knowledge Subdirectory:** Specify the location of custom knowledge files to enhance the agent's understanding. + +See the [Agent Profiles guide](../guides/agent-profiles.md) for profile +switching, the Easy and Advanced editors, capability policy, and project +availability. + +> [!NOTE] +> Since v0.9.7, custom prompts belong inside a specific agent profile rather +> than a shared `/prompts` folder. Most users should use **Manage agents** and +> the Agent Editor instead of editing files directly. + +> [!NOTE] +> The Hacker profile is included in the main image. After launch, choose the **hacker** agent profile in Settings to make it the default for new chats, or switch the selected chat from the composer profile selector. The "hacker" branch is deprecated. + +![settings](../res/setup/settings/1-agentConfig.png) + +### Chat Model Settings + +- **Provider:** Select the chat model provider (e.g., Anthropic) +- **Model Name:** Choose the specific model (e.g., claude-sonnet-4-5) +- **Context Length:** Set the maximum token limit for context window +- **Context Window Space:** Configure how much of the context window is dedicated to chat history + +![chat model settings](../res/setup/settings/2-chat-model.png) + +**Model naming is provider-specific.** + +Use `claude-sonnet-4-5` for Anthropic, but use `anthropic/claude-sonnet-4-5` for OpenRouter. If you see "Invalid model ID," verify the provider and naming format on the provider website, or search the web for " model naming". + +> [!TIP] +> **Context window tuning:** Set the total context window size first (for example, 100k), then adjust the chat history portion as a fraction of that total. A large fraction on a very large context window can still be enormous. + +> [!TIP] +> **API URL:** URL of the API endpoint for the chat model - only needed for some providers like Ollama, LM Studio, Azure, etc. + +### Utility Model Configuration + +- **Provider & Model:** Select a model for utility tasks like memory organization and summarization +- **Temperature:** Adjust the determinism of utility responses + +> [!NOTE] +> Utility models need to be strong enough to extract and consolidate memory reliably. Very small models (e.g., 4B) often fail at this; 70B-class models or high-quality cloud "flash/mini" models work best. + +### Embedding Model Settings [Optional] + +- **Provider:** Choose the embedding model provider (e.g., OpenAI) +- **Model Name:** Select the specific embedding model (e.g., text-embedding-3-small) + +> [!NOTE] +> Agent Zero uses a local embedding model by default (runs on CPU), but you can switch to OpenAI embeddings like `text-embedding-3-small` or `text-embedding-3-large` if preferred. + +### Built-in Voice Plugins + +- Agent Zero ships Whisper STT as the built-in `_whisper_stt` plugin and Kokoro TTS as the built-in `_kokoro_tts` plugin. +- Docker/bootstrap remains responsible for installing the required speech dependencies such as `ffmpeg`, Kokoro, Whisper, and `soundfile`. +- Both plugins can be enabled or disabled independently from the Agent Plugins section in the Web UI. +- Whisper model size, language, message handling, and silence behavior are configured from the plugin settings screen. +- If `_kokoro_tts` is disabled, spoken output falls back to the browser's native speech synthesis instead of the container runtime. + +### API Keys + +Configure API keys for various service providers directly within the Web UI. Click `Save` to confirm your settings. + +> [!NOTE] +> **OpenAI API vs Plus subscription:** A ChatGPT Plus subscription does not include API credits. You must provide a separate API key for OpenAI usage in Agent Zero. + +> [!TIP] +> For OpenAI-compatible providers (e.g., custom gateways or Z.AI/GLM), add the API key under **External Services -> Other OpenAI-compatible API keys**, then select **OpenAI Compatible** as the provider in model settings. + +> [!CAUTION] +> **GitHub Copilot Provider:** When using the GitHub Copilot provider, after selecting the model and entering your first prompt, the OAuth login procedure will begin. You'll find the authentication code and link in the output logs. Complete the authentication process by following the provided link and entering the code, then you may continue using Agent Zero. + +### Authentication + +- **UI Login:** Set username for web interface access +- **UI Password:** Configure password for web interface security +- **Root Password:** Manage Docker container root password for SSH access + +![settings](../res/setup/settings/3-auth.png) + +### Development Settings + +- **RFC Parameters (local instances only):** Configure URLs and ports for remote function calls between instances +- **RFC Password:** Configure password for remote function calls + +Learn more about Remote Function Calls in the [Development Setup guide](dev-setup.md#step-6-configure-ssh-and-rfc-connection). + +> [!IMPORTANT] +> Always keep your API keys and passwords secure. + +> [!NOTE] +> On Windows host installs (non-Docker), you must use RFC to run shell code on the host system. The Docker runtime handles this automatically. + +--- + +## Choosing Your LLMs + +The Settings page is the control center for selecting the Large Language Models (LLMs) that power Agent Zero. You can choose different LLMs for different roles: + +| LLM Role | Description | +| --- | --- | +| `chat_llm` | This is the primary LLM used for conversations, agent reasoning, and tool use. Vision support controls image understanding. | +| `utility_llm` | This LLM handles internal tasks like summarizing messages, managing memory, and processing internal prompts. Using a smaller, less expensive model here can improve efficiency. | +| `embedding_llm` | The embedding model shipped with A0 runs on CPU and is responsible for generating embeddings used for memory retrieval and knowledge base lookups. Changing the `embedding_llm` will re-index all of A0's memory. | + +**How to Change:** + +1. Open Settings page in the Web UI. +2. Choose the provider for the LLM for each role (Main Model, Utility Model, Embedding Model) and write the model name. +3. Click "Save" to apply the changes. + +> [!NOTE] +> The built-in Browser does not have a separate default model slot. The main agent decides when to call the direct `browser` tool. Browser settings can optionally choose a Browser LLM preset for Browser-owned helper operations. + +### Important Considerations + +#### Model Naming by Provider + +Use the naming format required by your selected provider: + +| Provider | Model Name Format | Example | +| --- | --- | --- | +| OpenAI | Model name only | `claude-sonnet-4-5` | +| OpenRouter | Provider prefix mostly required | `anthropic/claude-sonnet-4-5` | +| Ollama | Model name only | `gpt-oss:20b` | +| oMLX | API-visible model name from `/v1/models` | `Qwen3-0.6B-4bit` | +| llama.cpp | API-visible model name from `/v1/models` or `--alias` | `local-gguf` | +| vLLM | Hugging Face model ID or served model alias | `Qwen/Qwen2.5-1.5B-Instruct` | + +> [!TIP] +> If you see "Invalid model ID," verify the provider and naming format on the provider website, or search the web for " model naming". + +#### Local Model Server Addresses From Docker + +When Agent Zero runs in Docker, `localhost` and `127.0.0.1` inside an API base URL mean the Agent Zero container, not your host machine. For a model server running on the host, use `http://host.docker.internal:` when available, or the Docker host gateway address such as `http://172.17.0.1:` on the default Linux bridge. + +If the model server only listens on host loopback, for example `127.0.0.1:`, the container still cannot reach it through the gateway. Configure the local server to listen on a Docker-reachable address such as `0.0.0.0`, and keep that port limited to trusted clients. + +#### Context Window & Memory Split + +- Set the **total context window** (e.g., 100k) first. +- Then tune the **chat history portion** as a fraction of that total. +- Extremely large totals can make even small fractions very large; adjust thoughtfully. + +#### Utility Model Guidance + +- Utility models handle summarization and memory extraction. +- Very small models (about 4B) usually fail at reliable context extraction. +- Aim for ~70B class models or strong cloud "flash/mini" models for better results. + +#### Reasoning/Thinking Models + +- Reasoning can increase cost and latency. Some models perform better **without** reasoning. +- If a model supports it, disable reasoning via provider-specific parameters (e.g., Venice `disable_thinking=true`). + +--- + +## Installing and Using oMLX (Apple Silicon Local Models) + +oMLX is a local inference server for Apple Silicon Macs. It serves MLX models through an OpenAI-compatible API and supports chat, embeddings, and model listing endpoints. + +> [!NOTE] +> oMLX requires Apple Silicon and macOS 15+. On 16 GB machines, start with small quantized MLX models. + +### macOS oMLX Installation + +**Using Homebrew:** + +```bash +brew tap jundot/omlx https://github.com/jundot/omlx +brew install omlx +omlx start +``` + +**Using the macOS App:** + +Download the oMLX app from the [official website](https://omlx.ai/) and follow the welcome flow to choose a model directory, start the server, and download or discover models. + +By default, oMLX serves its OpenAI-compatible API at `http://localhost:8000/v1`. + +To run a foreground server with oMLX's paged SSD cache enabled: + +```bash +omlx serve --model-dir ~/.omlx/models --paged-ssd-cache-dir ~/.omlx/cache +``` + +### Configuring oMLX in Agent Zero + +1. Start oMLX and make sure at least one model is available in the oMLX dashboard or model directory. +2. In Agent Zero Settings, choose **oMLX** as the Chat model, Utility model, or Embedding model provider. +3. Use the model name shown by oMLX's model list or dashboard. +4. Agent Zero includes Docker-friendly defaults for oMLX on the host at `http://host.docker.internal:8000/v1`. Override the API base URL only if your oMLX server runs somewhere else. +5. Click `Save` to confirm your settings. + +> [!NOTE] +> If Agent Zero runs in Docker and oMLX runs on the Mac host, ensure port **8000** is reachable from the container. The shipped Docker Compose file maps `host.docker.internal` to the host gateway for Linux Docker. Docker Desktop for macOS provides this hostname automatically. + +--- + +## Installing and Using llama.cpp (GGUF Local Models) + +llama.cpp provides `llama-server`, a lightweight OpenAI-compatible HTTP server for GGUF models. Agent Zero talks to it through the same `/v1` API used by OpenAI-compatible clients. + +### macOS llama.cpp Installation + +**Using Homebrew:** + +```bash +brew install llama.cpp +``` + +Start a server with a downloaded GGUF model: + +```bash +llama-server -m ~/models/model.gguf --port 8080 --alias local-gguf +``` + +By default, Agent Zero expects llama.cpp at `http://host.docker.internal:8080/v1`. The model name can be the model path returned by `/v1/models`, but using `--alias` gives you a short stable name such as `local-gguf`. + +### Configuring llama.cpp in Agent Zero + +1. Start `llama-server` and confirm `http://localhost:8080/v1/models` returns your model. +2. In Agent Zero Settings, choose **llama.cpp** as the Chat model, Utility model, or Embedding model provider. +3. Use the model ID shown by `/v1/models`, or the alias you passed with `--alias`. +4. Override the API base URL only if you started `llama-server` on another host or port. +5. Click `Save` to confirm your settings. + +> [!NOTE] +> If Agent Zero runs in Docker and cannot reach a host-side `llama-server`, start the server on an address Docker can reach, for example `--host 0.0.0.0`, and keep the port firewalled to trusted clients. + +--- + +## Installing and Using vLLM (Local OpenAI-Compatible Serving) + +vLLM is a high-throughput local inference server with an OpenAI-compatible API. It is most common on Linux GPU hosts, and can also run on Apple Silicon through the vLLM Apple Silicon path or vLLM-Metal. + +For Apple Silicon Macs, install and activate vLLM-Metal: + +```bash +curl -fsSL https://raw.githubusercontent.com/vllm-project/vllm-metal/main/install.sh | bash +source ~/.venv-vllm-metal/bin/activate +``` + +Start a basic OpenAI-compatible server: + +```bash +vllm serve Qwen/Qwen2.5-1.5B-Instruct --host 0.0.0.0 --port 8000 +``` + +By default, Agent Zero expects vLLM at `http://host.docker.internal:8000/v1`, matching vLLM's default HTTP port. If another local provider already uses port 8000, start vLLM on another port and update Agent Zero's API base, for example `http://host.docker.internal:8001/v1`. + +### Configuring vLLM in Agent Zero + +1. Start vLLM and confirm `http://localhost:8000/v1/models` returns the served model. +2. In Agent Zero Settings, choose **vLLM** as the Chat model, Utility model, or Embedding model provider. +3. Use the model ID returned by vLLM's model list endpoint. +4. If you started vLLM with `--api-key`, enter the same key in the advanced provider settings or environment. +5. Click `Save` to confirm your settings. + +> [!NOTE] +> vLLM serves one model at a time by default. Use a generation model for Chat and Utility slots, and a separate embedding-capable vLLM server if you want vLLM embeddings. + +--- + +## Installing and Using Ollama (Local Models) + +Ollama is a powerful tool that allows you to run various large language models locally. + +--- + + +### Windows Windows Ollama Installation + +Download and install Ollama from the official website: + + + +Once installed, continue to [Pulling Models](#pulling-models). + +--- + + +### macOS macOS Ollama Installation + +**Using Homebrew:** + +```bash +brew install ollama +``` + +**Using Installer:** + +Download from the [official website](https://ollama.com/). + +Once installed, continue to [Pulling Models](#pulling-models). + +--- + + +### Linux Linux Ollama Installation + +Run the installation script: + +```bash +curl -fsSL https://ollama.com/install.sh | sh +``` + +Once installed, continue to [Pulling Models](#pulling-models). + +--- + +### Pulling Models + +**Finding Model Names:** + +Visit the [Ollama model library](https://ollama.com/library) for a list of available models and their corresponding names. Ollama models are referenced by **model name only** (for example, `llama3.2`). + +**Pull a model:** + +```bash +ollama pull +``` + +Replace `` with the name of the model you want to use. For example: `ollama pull mistral-large` + +### Configuring Ollama in Agent Zero + +1. Once you've downloaded your model(s), select it in the Settings page of the GUI. +2. Within the Chat model, Utility model, or Embedding model section, choose **Ollama** as provider. +3. Write your model code as expected by Ollama, in the format `llama3.2` or `qwen2.5:7b` +4. Agent Zero includes Docker-friendly defaults for Ollama on the host at `http://host.docker.internal:11434`. Override the API base URL only if your Ollama server runs somewhere else. +5. Click `Save` to confirm your settings. + +![ollama](../res/setup/settings/4-local-models.png) + +> [!NOTE] +> If Agent Zero runs in Docker and Ollama runs on the host, ensure port **11434** is reachable from the container. The shipped Docker Compose file maps `host.docker.internal` to the host gateway for Linux Docker. If both services are in the same Docker network, you can use `http://:11434` instead of `host.docker.internal`. + +### Managing Downloaded Models + +**Listing downloaded models:** + +```bash +ollama list +``` + +**Removing a model:** + +```bash +ollama rm +``` + +> [!TIP] +> Experiment with different model combinations to find the balance of performance and cost that best suits your needs. E.g., faster and lower latency LLMs will help, and you can also use `faiss_gpu` instead of `faiss_cpu` for the memory. + +--- + +## Using Agent Zero on Your Mobile Device + +Agent Zero can be accessed from mobile devices and other computers using the built-in **Tunnel feature**. + +### Recommended: Using Tunnel (Remote Access) + +The Tunnel feature allows secure access to your Agent Zero instance from anywhere: + +1. Open Settings in the Web UI +2. Navigate to the **External Services** tab +3. Click on **Flare Tunnel** in the navigation menu +4. Click **Create Tunnel** to generate a secure HTTPS URL +5. Share this URL to access Agent Zero from any device + +> [!IMPORTANT] +> **Security:** Always set a username and password in Settings -> Authentication before creating a tunnel to secure your instance on the internet. + +For complete details on tunnel configuration and security considerations, see the [Remote Access via Tunneling](../guides/usage.md#remote-access-via-tunneling) section in the Usage Guide. + +### Alternative: Local Network Access + +If you prefer to keep access limited to your local network: + +1. Find the mapped port in Docker Desktop (format: `:80`, e.g., `32771:80`) +2. Access from the same computer: `http://localhost:` +3. Access from other devices on the network: `http://:` + +> [!TIP] +> Find your computer's IP address with `ipconfig` (Windows) or `ifconfig`/`ip addr` (macOS/Linux). It's usually in the format `192.168.x.x` or `10.0.x.x`. + +For developers or users who need to run Agent Zero directly on their system, see the [In-Depth Guide for Full Binaries Installation](dev-setup.md). + +--- + +## Advanced: Automated Configuration via Environment Variables + +Agent Zero settings can be automatically configured using environment variables with the `A0_SET_` prefix in your `.env` file. This enables automated deployments without manual configuration. + +**Usage:** + +Add variables to your `.env` file in the format: + +```env +A0_SET_{setting_name}={value} +``` + +**Examples:** + +```env +# Model configuration +A0_SET_chat_model_provider=anthropic +A0_SET_chat_model_name=claude-3-5-sonnet-20241022 +A0_SET_chat_model_ctx_length=200000 + +# Memory settings +A0_SET_memory_recall_enabled=true +A0_SET_memory_recall_interval=5 + +# Agent configuration +A0_SET_agent_profile=custom +A0_SET_agent_memory_subdir=production +``` + +**Docker usage:** + +When running Docker, you can pass these as environment variables: + +```bash +docker run -p 50080:80 \ + -e A0_SET_chat_model_provider=anthropic \ + -e A0_SET_chat_model_name=claude-3-5-sonnet-20241022 \ + agent0ai/agent-zero +``` + +**Notes:** + +- These provide initial default values when settings.json doesn't exist or when new settings are added to the application. Once a value is saved in settings.json, it takes precedence over these environment variables. +- Sensitive settings (API keys, passwords) use their existing environment variables +- Container/process restart required for changes to take effect + +--- + +### Manual Migration (Legacy or Non-Docker) + +If you are migrating from older, non-Docker setups, A0 handles the migration of legacy folders and files automatically at runtime. The right place to save your files and directories is `a0/usr`. + +## Conclusion + +After following the instructions for your specific operating system, you should have Agent Zero successfully installed and running. You can now start exploring the framework's capabilities and experimenting with creating your own intelligent agents. + +**Next Steps:** + +- For production server deployments, see the [VPS Deployment Guide](vps-deployment.md) +- For development setup and extensions, see the [Development Setup Guide](dev-setup.md) +- For remote access via tunnel, see [Remote Access via Tunneling](../guides/usage.md#remote-access-via-tunneling) + +If you encounter any issues during the installation process, please consult the [Troubleshooting section](../guides/troubleshooting.md) of this documentation or refer to the Agent Zero [Skool](https://www.skool.com/agent-zero) or [Discord](https://discord.gg/B8KZKNsPpj) community for assistance. diff --git a/docs/setup/vps-deployment.md b/docs/setup/vps-deployment.md index e990a0be5e80346768d96110aba66991a35a6707..74b03f5d6d7205631d0bff18fd817fe887b60a3c 100644 --- a/docs/setup/vps-deployment.md +++ b/docs/setup/vps-deployment.md @@ -1,3 +1,770 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:35d128f65e9196f4a7714c2eca191b2a2535049356ad8c7e9a30c60d021068b1 -size 18115 +# Agent Zero Installation Guide + +> **Purpose:** Step-by-step guide for deploying Agent Zero instances on VPS/dedicated servers +> **Author:** Auto-generated from deployment experience +> **Last Updated:** December 21 2025 +> **Compatibility:** Docker-capable Linux servers (AlmaLinux, CentOS, Rocky, Ubuntu, Debian) + +--- + +## Table of Contents + +1. [Prerequisites](#prerequisites) +2. [Docker Installation](#docker-installation) +3. [Agent Zero Container Deployment](#agent-zero-container-deployment) +4. [Apache Reverse Proxy Configuration](#apache-reverse-proxy-configuration) +5. [SSL/TLS Configuration](#ssltls-configuration) +6. [Authentication Setup](#authentication-setup) +7. [Domain & DNS Setup](#domain-dns-setup) +8. [Verification & Testing](#verification-testing) +9. [Troubleshooting](#troubleshooting) +10. [Maintenance & Updates](#maintenance-updates) +11. [Quick Reference](#quick-reference) + +--- + +## Prerequisites + +### Server Requirements + +| Requirement | Minimum | Recommended | +|-------------|---------|-------------| +| **RAM** | 2 GB | 4+ GB | +| **Storage** | 20 GB | 50+ GB | +| **CPU** | 1 vCPU | 2+ vCPU | +| **OS** | Linux (64-bit) | AlmaLinux 9, Ubuntu 22.04+ | +| **Network** | Static IP | Dedicated IP with reverse DNS | + +### Required Access + +- Root or sudo access to the server +- SSH access (preferably on non-standard port) +- Domain/subdomain with DNS control +- SSL certificate (Let's Encrypt or commercial) + +### Software Dependencies + +- Docker Engine 24.0+ +- Apache 2.4+ with mod_proxy, mod_proxy_http, mod_proxy_wstunnel, mod_ssl, mod_rewrite +- curl, git (optional) + +--- + +## Docker Installation + +> [!NOTE] +> For detailed Docker installation instructions and alternative methods, see the [Linux Installation section](installation.md#linux-installation) in the main installation guide. + +### Method A: Debian/Ubuntu Systems + +```bash +# Update package index +apt-get update + +# Install prerequisites +apt-get install -y ca-certificates curl gnupg + +# Add Docker's official GPG key +install -m 0755 -d /etc/apt/keyrings +curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg +chmod a+r /etc/apt/keyrings/docker.gpg + +# Set up repository +echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null + +# Install Docker +apt-get update +apt-get install -y docker-ce docker-ce-cli containerd.io docker-compose-plugin + +# Start and enable Docker +systemctl enable docker +systemctl start docker +``` + +### Method B: AlmaLinux/Rocky/CentOS/RHEL Systems + +```bash +# Install required packages +dnf -y install dnf-plugins-core + +# Add Docker repository (use CentOS repo for AlmaLinux/Rocky) +dnf config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo + +# Install Docker +dnf -y install docker-ce docker-ce-cli containerd.io docker-compose-plugin + +# Start and enable Docker +systemctl enable docker +systemctl start docker +``` + +### Method C: Generic (Convenience Script) + +> ⚠️ **Note:** May not work on all distributions (e.g., AlmaLinux) + +```bash +curl -fsSL https://get.docker.com -o get-docker.sh +sh get-docker.sh +systemctl enable docker +systemctl start docker +``` + +### Verify Docker Installation + +```bash +docker --version +docker run hello-world +``` + +--- + +## Agent Zero Container Deployment + +### Step 1: Create Directory Structure + +```bash +# Choose your installation path +A0_NAME="a0-instance" # Change this to your instance name +A0_PATH="/opt/${A0_NAME}" + +# Create directories +mkdir -p ${A0_PATH} +mkdir -p ${A0_PATH}/work_dir +mkdir -p ${A0_PATH}/memory +mkdir -p ${A0_PATH}/logs +``` + +### Step 2: Create Environment Configuration + +```bash +# Create .env file with authentication +cat > ${A0_PATH}/.env << 'EOF' +# Agent Zero Configuration +# Authentication (REQUIRED for web access) +AUTH_LOGIN=your_username_here +AUTH_PASSWORD=your_secure_password_here + +# Optional: Additional configuration +# See Agent Zero documentation for all options +EOF +``` + +> ⚠️ **CRITICAL:** `AUTH_LOGIN` is the **username**, not a boolean! +> - ✅ Correct: `AUTH_LOGIN=admin` +> - ❌ Wrong: `AUTH_LOGIN=true` + +### Step 3: Choose Host Port + +| Port | Use Case | +|------|----------| +| `50080` | Standard/recommended for reverse proxy setups | +| `50081`, `50082`... | Additional instances on same server | +| `80` | Direct access (not recommended for production) | + +### Step 4: Pull and Run Container + +```bash +# Set variables +A0_NAME="a0-instance" +A0_PATH="/opt/${A0_NAME}" +A0_PORT="50080" + +# Pull latest image +docker pull agent0ai/agent-zero:latest + +# Run container +docker run -d --name ${A0_NAME} --restart unless-stopped -p ${A0_PORT}:80 -v ${A0_PATH}/.env:/a0/.env -v ${A0_PATH}/usr:/a0/usr agent0ai/agent-zero:latest +``` + +### Step 5: Verify Container + +```bash +# Check container is running +docker ps | grep ${A0_NAME} + +# Check logs +docker logs ${A0_NAME} + +# Test local access +curl -I http://127.0.0.1:${A0_PORT}/ +``` + +Expected response: `HTTP/1.1 302 FOUND` with `Location: /login` (if auth enabled) + +--- + +## Apache Reverse Proxy Configuration + +### Required Apache Modules + +```bash +# Debian/Ubuntu +a2enmod proxy proxy_http proxy_wstunnel ssl rewrite headers +systemctl restart apache2 + +# AlmaLinux/CentOS (usually pre-loaded) +httpd -M | grep -E "proxy|rewrite|ssl" +``` + +### Configuration for Standard Apache (Debian/Ubuntu) + +Create `/etc/apache2/sites-available/a0-instance.conf`: + +```apache +# Agent Zero Reverse Proxy Configuration +# Instance: a0-instance +# Domain: a0.example.com + +# HTTP - Redirect to HTTPS + + ServerName a0.example.com + ServerAlias www.a0.example.com + + RewriteEngine On + RewriteCond %{HTTPS} off + RewriteRule ^(.*)$ https://%{HTTP_HOST}%{REQUEST_URI} [L,R=301] + + +# HTTPS - Proxy to Container + + ServerName a0.example.com + ServerAlias www.a0.example.com + ServerAdmin webmaster@example.com + + # SSL Configuration + SSLEngine on + SSLCertificateFile /path/to/certificate.crt + SSLCertificateKeyFile /path/to/private.key + SSLCertificateChainFile /path/to/chain.crt + + # Proxy Configuration + ProxyPreserveHost On + ProxyPass / http://127.0.0.1:50080/ + ProxyPassReverse / http://127.0.0.1:50080/ + + # WebSocket Support (Required for real-time features) + RewriteEngine On + RewriteCond %{HTTP:Upgrade} websocket [NC] + RewriteCond %{HTTP:Connection} upgrade [NC] + RewriteRule ^/?(.*) ws://127.0.0.1:50080/$1 [P,L] + + # Logging + ErrorLog ${APACHE_LOG_DIR}/a0-instance.error.log + CustomLog ${APACHE_LOG_DIR}/a0-instance.access.log combined + +``` + +Enable and restart: + +```bash +a2ensite a0-instance.conf +apache2ctl configtest +systemctl reload apache2 +``` + +### Configuration for DirectAdmin Apache (AlmaLinux/CentOS) + +#### Option A: Use httpd-includes.conf (Recommended) + +Edit `/etc/httpd/conf/extra/httpd-includes.conf`: + +```apache +# Agent Zero Proxy Configuration +# Instance: a0-instance +# Domain: a0.example.com +# Note: Use specific IP, not wildcards, for DirectAdmin compatibility + + + ServerName a0.example.com + ServerAlias www.a0.example.com + + RewriteEngine On + RewriteCond %{HTTPS} off + RewriteRule ^(.*)$ https://%{HTTP_HOST}%{REQUEST_URI} [L,R=301] + + + + ServerName a0.example.com + ServerAlias www.a0.example.com + ServerAdmin webmaster@example.com + + SSLEngine on + # DirectAdmin SSL cert paths (adjust user and domain) + SSLCertificateFile /usr/local/directadmin/data/users/USERNAME/domains/example.com.cert.combined + SSLCertificateKeyFile /usr/local/directadmin/data/users/USERNAME/domains/example.com.key + + ProxyPreserveHost On + ProxyPass / http://127.0.0.1:50080/ + ProxyPassReverse / http://127.0.0.1:50080/ + + # WebSocket Support + RewriteEngine On + RewriteCond %{HTTP:Upgrade} websocket [NC] + RewriteCond %{HTTP:Connection} upgrade [NC] + RewriteRule ^/?(.*) ws://127.0.0.1:50080/$1 [P,L] + + ErrorLog /var/log/httpd/domains/a0.example.com.error.log + CustomLog /var/log/httpd/domains/a0.example.com.access.log combined + +``` + +> ⚠️ **Important for DirectAdmin:** +> - Use **specific IP address** (e.g., `192.168.1.100:443`), not `*:443` +> - IP-bound vhosts take precedence over DirectAdmin's vhosts +> - SSL certs are in `/usr/local/directadmin/data/users/USERNAME/domains/` + +#### Option B: Standalone conf.d file + +If `/etc/httpd/conf.d/` is included in your Apache config: + +```bash +# Check if conf.d is included +grep 'conf.d' /etc/httpd/conf/httpd.conf + +# If not, add before directadmin-vhosts.conf include: +sed -i '/Include conf\/extra\/directadmin-vhosts.conf/i Include conf.d/*.conf' /etc/httpd/conf/httpd.conf + +# Create config +mkdir -p /etc/httpd/conf.d +cat > /etc/httpd/conf.d/httpd-vhosts-a0.conf << 'EOF' +# Your vhost config here (same as Option A) +EOF +``` + +### Verify and Restart Apache + +```bash +# Test configuration +httpd -t # AlmaLinux/CentOS +apachectl -t # Alternative +apache2ctl -t # Debian/Ubuntu + +# Restart +systemctl restart httpd # AlmaLinux/CentOS +systemctl restart apache2 # Debian/Ubuntu +``` + +--- + +## SSL/TLS Configuration + +### Option A: Let's Encrypt with Certbot + +```bash +# Install Certbot +# Debian/Ubuntu: +apt-get install certbot python3-certbot-apache + +# AlmaLinux/CentOS: +dnf install certbot python3-certbot-apache + +# Obtain certificate +certbot --apache -d a0.example.com -d www.a0.example.com + +# Auto-renewal (usually automatic, but verify) +certbot renew --dry-run +``` + +### Option B: DirectAdmin Auto-SSL + +If using DirectAdmin, SSL is typically managed automatically: + +1. Create domain/subdomain in DirectAdmin +2. Enable "SSL" for the domain +3. DirectAdmin will obtain Let's Encrypt certificate +4. Certs stored in `/usr/local/directadmin/data/users/USERNAME/domains/` + +### Option C: Manual/Commercial Certificates + +Place certificates in secure location: + +```bash +mkdir -p /etc/ssl/a0 +chmod 700 /etc/ssl/a0 + +# Copy your certificates +cp certificate.crt /etc/ssl/a0/ +cp private.key /etc/ssl/a0/ +cp chain.crt /etc/ssl/a0/ # if applicable + +chmod 600 /etc/ssl/a0/* +``` + +--- + +## Authentication Setup + +### Understanding A0 Authentication Variables + +| Variable | Purpose | Example | +|----------|---------|--------| +| `AUTH_LOGIN` | The **username** for login | `AUTH_LOGIN=admin` | +| `AUTH_PASSWORD` | The **password** for login | `AUTH_PASSWORD=SecurePass123!` | + +> ⚠️ **Common Mistake:** `AUTH_LOGIN` is the username, **not** a boolean to enable auth! + +### Setting Up Authentication + +```bash +# Edit .env file +vi /opt/a0-instance/.env + +# Add/update these lines: +AUTH_LOGIN=your_username +AUTH_PASSWORD=your_secure_password + +# Restart container to apply +docker restart a0-instance +``` + +### Password Requirements + +- Minimum 8 characters recommended +- Special characters are supported (properly escaped) +- Avoid these characters in passwords: `' " \` $ \` (or escape carefully) + +### Disabling Authentication (Not Recommended) + +To disable authentication (local/dev use only): + +```bash +# Remove or comment out both lines in .env: +# AUTH_LOGIN= +# AUTH_PASSWORD= + +docker restart a0-instance +``` + +--- + +## Domain & DNS Setup + +### DNS Configuration + +Create an A record pointing to your server: + +| Type | Name | Value | TTL | +|------|------|-------|-----| +| A | a0 | YOUR_SERVER_IP | 300 | +| A | www.a0 | YOUR_SERVER_IP | 300 | + +### DirectAdmin Subdomain Setup + +1. Log into DirectAdmin +2. Navigate to: **Domain Setup** → Select domain → **Subdomain Management** +3. Create subdomain (e.g., `a0`) +4. Note: You'll override the DocumentRoot with Apache proxy config + +### Verify DNS Propagation + +```bash +# Check DNS resolution +dig a0.example.com +short +nslookup a0.example.com + +# Should return your server IP +``` + +--- + +## Verification & Testing + +### Step-by-Step Verification Checklist + +```bash +# 1. Verify Docker container is running +docker ps | grep a0-instance + +# 2. Check container logs for errors +docker logs a0-instance --tail 50 + +# 3. Test local container access +curl -I http://127.0.0.1:50080/ +# Expected: HTTP/1.1 302 FOUND, Location: /login + +# 4. Test Apache config +httpd -t # or apache2ctl -t + +# 5. Check Apache is proxying correctly +curl -I http://127.0.0.1:80 -H "Host: a0.example.com" +curl -Ik https://127.0.0.1:443 -H "Host: a0.example.com" + +# 6. Test external HTTPS access +curl -I https://a0.example.com/ +# Expected: HTTP/2 302 with Location: /login + +# 7. Test login page loads +curl -s https://a0.example.com/login | grep -i "" +# Expected: <title>Login - Agent Zero +``` + +### WebSocket Verification + +```bash +# Install wscat if needed +npm install -g wscat + +# Test WebSocket connection +wscat -c wss://a0.example.com/ws +``` + +--- + +## Troubleshooting + +### Issue: "Invalid Credentials" on Login + +**Cause:** Incorrect `.env` configuration + +**Fix:** +```bash +# Verify .env inside container +docker exec a0-instance cat /a0/.env + +# Ensure format is: +# AUTH_LOGIN=username (NOT AUTH_LOGIN=true) +# AUTH_PASSWORD=password + +# Restart after fixing +docker restart a0-instance +``` + +### Issue: 403 Forbidden + +**Cause:** DirectAdmin vhost overriding custom proxy config + +**Fix:** +```bash +# Check vhost order +httpd -S 2>&1 | grep your-domain + +# Ensure custom config loads BEFORE directadmin-vhosts.conf +# Use specific IP binding (e.g., 192.168.1.1:443) not wildcards (*:443) + +# Restart Apache +systemctl restart httpd +``` + +### Issue: 502 Bad Gateway + +**Cause:** Container not running or wrong port + +**Fix:** +```bash +# Check container status +docker ps -a | grep a0-instance + +# If stopped, check logs +docker logs a0-instance + +# Restart container +docker start a0-instance + +# Verify port binding +netstat -tlnp | grep 50080 +``` + +### Issue: 504 Gateway Timeout + +**Cause:** Container overloaded or unresponsive + +**Fix:** +```bash +# Check container resource usage +docker stats a0-instance --no-stream + +# Restart container +docker restart a0-instance + +# Check for memory issues +free -h +``` + +### Issue: WebSocket Connection Failed + +**Cause:** Missing WebSocket proxy rules + +**Fix:** +Ensure these lines are in your vhost config: + +```apache +RewriteEngine On +RewriteCond %{HTTP:Upgrade} websocket [NC] +RewriteCond %{HTTP:Connection} upgrade [NC] +RewriteRule ^/?(.*) ws://127.0.0.1:50080/$1 [P,L] +``` + +### Issue: Container Won't Start + +**Cause:** Port conflict or Docker issue + +**Fix:** +```bash +# Check what's using the port +netstat -tlnp | grep 50080 + +# Remove conflicting container +docker rm -f conflicting-container + +# Check Docker daemon +systemctl status docker +journalctl -u docker --since "1 hour ago" +``` + +### Issue: Changes to .env Not Taking Effect + +**Cause:** Container needs restart to reload env + +**Fix:** +```bash +docker restart a0-instance + +# Verify env is loaded +docker exec a0-instance cat /a0/.env +``` + +--- + +## Maintenance & Updates + +### Updating Agent Zero + +```bash +# Pull latest image +docker pull agent0ai/agent-zero:latest + +# Stop and remove old container (data persists in volumes) +docker stop a0-instance +docker rm a0-instance + +# Recreate with same settings +docker run -d --name a0-instance --restart unless-stopped -p 50080:80 -v /opt/a0-instance/.env:/a0/.env -v /opt/a0-instance/usr:/a0/usr -v /opt/agent-zero:latest +``` + +### Backup Strategy + +```bash +# Backup all instance data +tar -czvf a0-backup-$(date +%Y%m%d).tar.gz /opt/a0-instance/ + +# Key items to backup: +# - /opt/a0-instance/.env (configuration) +# - /opt/a0-instance/memory/ (agent memories) +# - /opt/a0-instance/work_dir/ (working files) +``` + +### Monitoring + +```bash +# Check container health +docker ps --format "table {{.Names}} {{.Status}} {{.Ports}}" + +# View recent logs +docker logs --tail 100 -f a0-instance + +# Resource usage +docker stats a0-instance +``` + +### Docker Cleanup + +```bash +# Remove unused images +docker image prune -f + +# Remove all unused Docker resources +docker system prune -f +``` + +--- + +## Quick Reference + +### Essential Commands + +```bash +# Container Management +docker start a0-instance +docker stop a0-instance +docker restart a0-instance +docker logs a0-instance +docker exec -it a0-instance bash + +# Apache Management +systemctl restart httpd # RHEL/AlmaLinux +systemctl restart apache2 # Debian/Ubuntu +httpd -t # Test config + +# Quick Diagnostics +docker ps | grep a0 +curl -I https://your-domain.com/login +``` + +### Standard Paths + +| Component | Path | +|-----------|----- | +| Instance Data | `/opt/a0-instance/` | +| Environment File | `/opt/a0-instance/.env` | +| Memory Storage | `/opt/a0-instance/memory/` | +| Work Directory | `/opt/a0-instance/work_dir/` | +| Apache Config (Standard) | `/etc/apache2/sites-available/` | +| Apache Config (DirectAdmin) | `/etc/httpd/conf/extra/httpd-includes.conf` | +| DirectAdmin SSL Certs | `/usr/local/directadmin/data/users/USER/domains/` | + +### Standard Ports + +| Port | Purpose | +|------|---------| +| 50080 | First A0 instance | +| 50081 | Second A0 instance | +| 50082 | Third A0 instance | +| 80 | HTTP (redirect to HTTPS) | +| 443 | HTTPS (main access) | + +### .env Template + +```bash +# Agent Zero Configuration Template +# Copy and customize for each instance + +# Authentication (REQUIRED for production) +AUTH_LOGIN=your_username +AUTH_PASSWORD=your_secure_password + +# Optional: Additional settings +# Refer to Agent Zero documentation for all options +``` + +--- + +## Appendix: Multi-Instance Setup + +For running multiple A0 instances on the same server: + +```bash +# Instance 1: a0-primary on port 50080 +mkdir -p /opt/a0-primary +# ... create .env, run container on port 50080 + +# Instance 2: a0-dev on port 50081 +mkdir -p /opt/a0-dev +# ... create .env, run container on port 50081 + +# Instance 3: a0-backup on port 50082 +mkdir -p /opt/a0-backup +# ... create .env, run container on port 50082 +``` + +Each instance needs: +- Unique container name +- Unique host port +- Separate data directory +- Separate domain/subdomain +- Separate Apache vhost config + +--- + +*This guide comes from successful Agent Zero deployments across DirectAdmin and standard Linux environments.* + +Contributed by @hurtdidit in the A0 Community. diff --git a/extensions/AGENTS.md b/extensions/AGENTS.md index 1af5895758881f263457c0ef0465cb484521a9dd..0658654cb70755fd5f728fb64df2f498bf068fd9 100644 --- a/extensions/AGENTS.md +++ b/extensions/AGENTS.md @@ -1,3 +1,39 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:2fee5d7084cc2d56ea3b981a3c921e2e7944bc20a20fd8691d5caf0b94b2d7f8 -size 1537 +# Extensions DOX + +## Purpose + +- Own core lifecycle extension implementations for backend and WebUI extension points. +- Keep built-in hook behavior ordered, discoverable, and compatible with plugin extension discovery. + +## Ownership + +- `python/` contains backend lifecycle hooks executed through `helpers.extension`. +- `webui/` contains frontend extension contributions loaded through `webui/js/extensions.js`. +- Plugin-specific extensions belong inside each plugin's `extensions/` directory. + +## Local Contracts + +- Extension directory names are runtime extension point names. +- File ordering matters when names include numeric prefixes. +- Extensions must be safe to run repeatedly when the lifecycle point can fire multiple times. +- Secret masking, auth, security, and persistence extensions must not be bypassed by convenience changes. + +## Work Guidance + +- Keep extension code small and focused on its hook point. +- Move shared logic into `helpers/` when it is reused outside one extension. +- Coordinate changes with plugin extension docs and tests when extension point semantics change. + +## Verification + +- Run targeted lifecycle, prompt, stream, WebSocket, or WebUI extension tests for changed hook points. +- Smoke-test startup when changing initialization, migration, or system-prompt extensions. + +## Child DOX Index + +Direct child DOX files: + +| Child | Scope | +| --- | --- | +| [python/AGENTS.md](python/AGENTS.md) | Backend lifecycle extension hook files. | +| [webui/AGENTS.md](webui/AGENTS.md) | Frontend extension contributions. | diff --git a/extensions/python/AGENTS.md b/extensions/python/AGENTS.md index 51c92ac45a1bc038e07a3c6b90aaf8bc2f9dbfff..5138c919c051883975dc8424a0ff024a43ad47fa 100644 --- a/extensions/python/AGENTS.md +++ b/extensions/python/AGENTS.md @@ -1,3 +1,68 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:a60e8fc1128128aafbb5a9a5b45033ab0d21af713353e168f3008c2276b71cac -size 4681 +# Python Extensions DOX + +## Purpose + +- Own built-in backend lifecycle extensions under `extensions/python/`. +- Keep Python hook behavior compatible with `helpers.extension.call_extensions_async` and `call_extensions_sync`. + +## Ownership + +- Each direct subdirectory is one named extension point. +- Python files inside an extension point are loaded in deterministic filename order. +- Implicit `@extensible` hook implementations use `_functions////` layout when present. + +## Local Contracts + +- Extension functions must match the arguments supplied by their hook point. +- Preserve numeric prefixes when ordering affects prompt construction, stream masking, persistence, or cleanup. +- Use `AgentContext` from `agent` when context access is needed. +- Do not log unmasked secrets, raw hidden prompt sections, or private user data. + +## Work Guidance + +- Keep extension modules import-light; many hooks run during hot paths. +- Use mutable `ctx` or `data` dictionaries according to the hook contract when rewriting content. +- Add or update tests when a hook changes prompt content, message history, tool output, streaming, or persistence behavior. + +## Verification + +- Run targeted tests for the affected lifecycle area. +- Run a startup smoke check for `agent_init`, `startup_migration`, or `system_prompt` changes when practical. + +## Child DOX Index + +Direct child DOX files: + +| Child | Scope | +| --- | --- | +| [_functions/AGENTS.md](_functions/AGENTS.md) | Implicit `@extensible` backend hook implementations. | +| [agent_init/AGENTS.md](agent_init/AGENTS.md) | Agent context initialization hooks. | +| [banners/AGENTS.md](banners/AGENTS.md) | Backend banner and discovery-card contributions. | +| [before_main_llm_call/AGENTS.md](before_main_llm_call/AGENTS.md) | Pre-main-model-call behavior. | +| [error_format/AGENTS.md](error_format/AGENTS.md) | Error formatting and masking behavior. | +| [hist_add_before/AGENTS.md](hist_add_before/AGENTS.md) | Pre-history-insertion masking behavior. | +| [hist_add_tool_result/AGENTS.md](hist_add_tool_result/AGENTS.md) | Tool-result history side effects. | +| [job_loop/AGENTS.md](job_loop/AGENTS.md) | Periodic backend maintenance jobs. | +| [message_loop_end/AGENTS.md](message_loop_end/AGENTS.md) | End-of-message-loop history and persistence behavior. | +| [message_loop_result/AGENTS.md](message_loop_result/AGENTS.md) | Completed model-result normalization and default-processing interception. | +| [message_loop_prompts_after/AGENTS.md](message_loop_prompts_after/AGENTS.md) | Prompt protocol and extras assembled around message-loop prompt construction. | +| [message_loop_prompts_before/AGENTS.md](message_loop_prompts_before/AGENTS.md) | Pre-prompt-construction message-loop gates. | +| [message_loop_start/AGENTS.md](message_loop_start/AGENTS.md) | Start-of-message-loop iteration state. | +| [monologue_end/AGENTS.md](monologue_end/AGENTS.md) | End-of-monologue UI and cleanup behavior. | +| [monologue_start/AGENTS.md](monologue_start/AGENTS.md) | Core start-of-monologue lifecycle extensions. | +| [process_chain_end/AGENTS.md](process_chain_end/AGENTS.md) | Process-chain completion and queued-message handling. | +| [reasoning_stream/AGENTS.md](reasoning_stream/AGENTS.md) | Full reasoning stream handling. | +| [reasoning_stream_chunk/AGENTS.md](reasoning_stream_chunk/AGENTS.md) | Reasoning stream chunk masking. | +| [reasoning_stream_end/AGENTS.md](reasoning_stream_end/AGENTS.md) | Reasoning stream finalization. | +| [response_stream/AGENTS.md](response_stream/AGENTS.md) | Full assistant response stream handling. | +| [response_stream_chunk/AGENTS.md](response_stream_chunk/AGENTS.md) | Assistant response chunk masking. | +| [response_stream_end/AGENTS.md](response_stream_end/AGENTS.md) | Assistant response stream finalization. | +| [startup_migration/AGENTS.md](startup_migration/AGENTS.md) | Startup migrations. | +| [system_prompt/AGENTS.md](system_prompt/AGENTS.md) | Core system prompt section construction. | +| [tool_execute_after/AGENTS.md](tool_execute_after/AGENTS.md) | Post-tool-execution processing. | +| [tool_execute_before/AGENTS.md](tool_execute_before/AGENTS.md) | Pre-tool-execution processing. | +| [user_message_ui/AGENTS.md](user_message_ui/AGENTS.md) | User-visible UI message hooks. | +| [util_model_call_before/AGENTS.md](util_model_call_before/AGENTS.md) | Pre-utility-model-call masking. | +| [webui_ws_connect/AGENTS.md](webui_ws_connect/AGENTS.md) | WebUI WebSocket connect behavior. | +| [webui_ws_disconnect/AGENTS.md](webui_ws_disconnect/AGENTS.md) | WebUI WebSocket disconnect behavior. | +| [webui_ws_event/AGENTS.md](webui_ws_event/AGENTS.md) | Incoming WebUI WebSocket event behavior. | diff --git a/extensions/python/_functions/AGENTS.md b/extensions/python/_functions/AGENTS.md index d5f9482461ab7bdf0ac5c4b870f76fd300a101eb..0b7954fdb98eb21ce2cfedee1ae6f27921e637b6 100644 --- a/extensions/python/_functions/AGENTS.md +++ b/extensions/python/_functions/AGENTS.md @@ -1,3 +1,33 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:45b0d49727663cfd9f6c3bf81431bde197789b2392924aa68bed1a8afae2c296 -size 1501 +# Python Function Extensions DOX + +## Purpose + +- Own implicit `@extensible` backend hook implementations. +- Preserve nested module, class/function, method, and `start`/`end` extension layout. + +## Ownership + +- Each nested path mirrors a Python module and qualname segment. +- Leaf `start/` and `end/` directories own ordered extension files for that extensible function point. + +## Local Contracts + +- Do not flatten nested qualname paths into retired legacy folder names. +- Extension functions must match the implicit hook's supplied arguments. +- Preserve ordering prefixes where exception handling, watchdog registration, or cleanup depends on them. +- Hooks that mirror persisted AI responses into UI logs must reuse existing stream log items and avoid duplicating live response-tool logs. +- The `AgentContext.run_task/end` hook attaches integration callbacks to the returned `DeferredTask`; keep terminal side effects out of `agent.py`. +- Recovery-loop circuit breakers must stop at the General Settings limit and render their user-visible cost warning from a core framework prompt. +- Prompt settings snapshots must be task-local, accessed through `get_settings_for_prompt()`, and end with the matching `Agent.prepare_prompt` call, including exceptional exits. + +## Work Guidance + +- Keep implicit hook extensions narrow and colocated with the exact function point they extend. + +## Verification + +- Run targeted tests for the affected function point after changes. + +## Child DOX Index + +No child DOX files. diff --git a/extensions/python/_functions/__main__/init_a0/end/_10_register_watchdogs.py b/extensions/python/_functions/__main__/init_a0/end/_10_register_watchdogs.py index 4e5b0c10ebb9212ae8c1131414152ed51238c840..e61001e03119bad2073c52aa237a5552a405d51f 100644 --- a/extensions/python/_functions/__main__/init_a0/end/_10_register_watchdogs.py +++ b/extensions/python/_functions/__main__/init_a0/end/_10_register_watchdogs.py @@ -1,3 +1,11 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:3585ef1b3cd71ea587ba5fed1ebdc4e37be8e6517bc888d4744938bc27561cdd -size 344 +from helpers.extension import Extension + + +class RegisterWatchDogs(Extension): + + def execute(self, **kwargs): + from helpers.plugins import register_watchdogs as register_plugins_watchdogs + from helpers.api import register_watchdogs as register_api_watchdogs + + register_plugins_watchdogs() + register_api_watchdogs() \ No newline at end of file diff --git a/extensions/python/_functions/agent/Agent/handle_exception/end/_40_handle_intervention_exception.py b/extensions/python/_functions/agent/Agent/handle_exception/end/_40_handle_intervention_exception.py index 4ee872948ed3301abfc7f012ce416888555a14c6..71437278c5f450598132a4833b501fb57978ba17 100644 --- a/extensions/python/_functions/agent/Agent/handle_exception/end/_40_handle_intervention_exception.py +++ b/extensions/python/_functions/agent/Agent/handle_exception/end/_40_handle_intervention_exception.py @@ -1,3 +1,21 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:04ee5f0807d0d8f697979f33c0ff2ba7049a85bbad3aa6955738a3a60b0336f9 -size 640 +from datetime import datetime, timezone +from helpers.extension import Extension +from agent import LoopData +from helpers.localization import Localization +from helpers.errors import InterventionException +from helpers import errors +from helpers.print_style import PrintStyle + + +class HandleInterventionException(Extension): + async def execute(self, data: dict = {}, **kwargs): + if not self.agent: + return + + if not data.get("exception"): + return + + if isinstance(data["exception"], InterventionException): + data["exception"] = None # skip the exception and continue message loop + + diff --git a/extensions/python/_functions/agent/Agent/handle_exception/end/_50_handle_repairable_exception.py b/extensions/python/_functions/agent/Agent/handle_exception/end/_50_handle_repairable_exception.py index 7eb09f6e6bcf80b4c53a42092d3d1a160b1a2f81..e6d098c2a0e886f3c2eae5d8ebb0c4c5b472ca84 100644 --- a/extensions/python/_functions/agent/Agent/handle_exception/end/_50_handle_repairable_exception.py +++ b/extensions/python/_functions/agent/Agent/handle_exception/end/_50_handle_repairable_exception.py @@ -1,3 +1,25 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:67e15a2835ee4e4d02f7f277a32a42273f6bddad4e8b2448f3588aec2bcc54b0 -size 991 +from datetime import datetime, timezone +from helpers.extension import Extension +from agent import LoopData +from helpers.localization import Localization +from helpers.errors import RepairableException +from helpers import errors, extension +from helpers.print_style import PrintStyle + +class HandleRepairableException(Extension): + async def execute(self, data: dict = {}, **kwargs): + if not self.agent: + return + + if not data.get("exception"): + return + + if isinstance(data["exception"], RepairableException): + msg = {"message": errors.format_error(data["exception"])} + await extension.call_extensions_async("error_format", agent=self.agent, msg=msg) + wmsg = self.agent.hist_add_warning(msg["message"]) + PrintStyle(font_color="red", padding=True).print(msg["message"]) + self.agent.context.log.log(type="warning", content=msg["message"], id=wmsg.id) + data["exception"] = None + + diff --git a/extensions/python/_functions/agent/Agent/handle_exception/end/_90_handle_critical_exception.py b/extensions/python/_functions/agent/Agent/handle_exception/end/_90_handle_critical_exception.py index 380a50366fa7008178dfeeec5340828fd3d0ad2d..ccd6edfc71096f2de0c3a61cdb69a01946a33b49 100644 --- a/extensions/python/_functions/agent/Agent/handle_exception/end/_90_handle_critical_exception.py +++ b/extensions/python/_functions/agent/Agent/handle_exception/end/_90_handle_critical_exception.py @@ -1,3 +1,43 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:762c8469756a147e96d7cbed07cb0d585d296006b10475f7fbe143e6ded43b32 -size 1500 +import asyncio + +from helpers.extension import Extension +from helpers.print_style import PrintStyle +from helpers import errors + +from helpers.errors import HandledException + + +class HandleCriticalException(Extension): + async def execute(self, data: dict = {}, **kwargs): + if not self.agent: + return + + if not (exception:= data.get("exception")): + return + + # when exception is HandledException, keep it active, no logging here + if isinstance(exception, HandledException): + return + + # asyncio cancel - chat is being terminated, print out and re-raise as handledException + if isinstance(exception, asyncio.CancelledError): + PrintStyle(font_color="white", background_color="red", padding=True).print( + f"Context {self.agent.context.id} terminated during message loop" + ) + data["exception"] = HandledException(exception) + return + + # other exceptions should be logged and re-raised as HandledException + error_text = errors.error_text(exception) + error_message = errors.format_error(exception) + + PrintStyle(font_color="red", padding=True).print(error_message) + self.agent.context.log.log( + type="error", + content=error_message, + ) + PrintStyle(font_color="red", padding=True).print( + f"{self.agent.agent_name}: {error_text}" + ) + + data["exception"] = HandledException(exception) diff --git a/extensions/python/_functions/agent/Agent/hist_add_ai_response/end/_10_log_plain_responses.py b/extensions/python/_functions/agent/Agent/hist_add_ai_response/end/_10_log_plain_responses.py index 3eea722cc698e9215ef007bd10b863fe79179a3a..28c80e77c543bb442f3a22059c632c1f8c32c42b 100644 --- a/extensions/python/_functions/agent/Agent/hist_add_ai_response/end/_10_log_plain_responses.py +++ b/extensions/python/_functions/agent/Agent/hist_add_ai_response/end/_10_log_plain_responses.py @@ -1,3 +1,47 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:cbb24ea2ec8c6acf3f4a7df4b6042464e4c3471118f865a843ca2d9f90b90950 -size 1498 +from typing import Any + +from helpers import extract_tools +from helpers.extension import Extension + + +class LogPlainResponses(Extension): + def execute(self, data: dict[str, Any] | None = None, **kwargs): + if not self.agent or not isinstance(data, dict): + return + + call_kwargs = data.get("kwargs") + if not isinstance(call_kwargs, dict): + call_kwargs = {} + + llm_result = call_kwargs.get("llm_result") + if getattr(llm_result, "mode", "") != "responses": + return + + message = call_kwargs.get("message") + call_args = data.get("args") + if message is None and isinstance(call_args, tuple) and len(call_args) > 1: + message = call_args[1] + if not isinstance(message, str) or not message: + return + if ( + extract_tools.extract_tool_request(message) is not None + or extract_tools.is_misformatted_tool_request(message) + ): + return + + params = getattr(getattr(self.agent, "loop_data", None), "params_temporary", None) + if not isinstance(params, dict) or "log_item_response" in params: + return + + log_item = params.get("log_item_generating") + if log_item is None: + return + + params["log_item_response"] = log_item + log_item.update( + type="response", + heading="", + content=message, + finished=True, + update_progress="none", + ) diff --git a/extensions/python/_functions/agent/Agent/hist_add_warning/end/_90_stop_unusable_response_loop.py b/extensions/python/_functions/agent/Agent/hist_add_warning/end/_90_stop_unusable_response_loop.py index fbffb7ad0ecbb808d80f0053987e136c02a137a5..87670a2731f7e7a23371861e09f77115865caebc 100644 --- a/extensions/python/_functions/agent/Agent/hist_add_warning/end/_90_stop_unusable_response_loop.py +++ b/extensions/python/_functions/agent/Agent/hist_add_warning/end/_90_stop_unusable_response_loop.py @@ -1,3 +1,56 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:7c138b131522d75756865ef2aebc656f12c7d16f0ce5feefc6c733a3916b1b84 -size 2012 +from helpers.errors import HandledException +from helpers.extension import Extension +from helpers.settings import get_settings + + +STATE_KEY = "_unusable_response_failures" + + +class StopUnusableResponseLoop(Extension): + def execute(self, data: dict | None = None, **kwargs): + if not self.agent or not isinstance(data, dict): + return + + call_kwargs = data.get("kwargs") + message = call_kwargs.get("message") if isinstance(call_kwargs, dict) else None + call_args = data.get("args") + if message is None and isinstance(call_args, tuple) and len(call_args) > 1: + message = call_args[1] + + if not isinstance(message, str): + return + if message not in { + self.agent.read_prompt("fw.msg_misformat.md"), + self.agent.read_prompt("fw.msg_repeat.md"), + self.agent.read_prompt("fw.msg_empty_response.md"), + }: + return + + loop_data = getattr(self.agent, "loop_data", None) + state = getattr(loop_data, "params_persistent", None) + iteration = getattr(loop_data, "iteration", None) + if not isinstance(state, dict) or not isinstance(iteration, int): + return + + previous = state.get(STATE_KEY, {}) + if not isinstance(previous, dict): + previous = {} + previous_iteration = previous.get("iteration") + if previous_iteration == iteration: + return + + count = ( + previous.get("count", 0) + 1 + if previous_iteration == iteration - 1 + else 1 + ) + state[STATE_KEY] = {"iteration": iteration, "count": count} + limit = get_settings()["max_consecutive_unusable_responses"] + if count < limit: + return + + stop_message = self.agent.read_prompt( + "fw.msg_unusable_response_limit.md", limit=limit + ) + self.agent.context.log.log(type="warning", content=stop_message) + data["exception"] = HandledException(stop_message) diff --git a/extensions/python/_functions/agent/Agent/prepare_prompt/end/_00_restore_settings.py b/extensions/python/_functions/agent/Agent/prepare_prompt/end/_00_restore_settings.py index 9cf7d94604519e0cb5331c25429d7463dd0d5b2e..6165340897a98d425d8e4429e32da10030ebe416 100644 --- a/extensions/python/_functions/agent/Agent/prepare_prompt/end/_00_restore_settings.py +++ b/extensions/python/_functions/agent/Agent/prepare_prompt/end/_00_restore_settings.py @@ -1,3 +1,12 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:02352e6a668d95ed2d72d77c3fe417f4ceea29a3ad7601666059c569d06e48e9 -size 385 +from helpers import settings +from helpers.extension import Extension + + +_TOKEN_KEY = "_prompt_settings_snapshot_token" + + +class RestorePromptSettings(Extension): + def execute(self, data: dict | None = None, **kwargs): + token = data.pop(_TOKEN_KEY, None) if isinstance(data, dict) else None + if token is not None: + settings.end_prompt_settings_snapshot(token) diff --git a/extensions/python/_functions/agent/Agent/prepare_prompt/start/_99_snapshot_settings.py b/extensions/python/_functions/agent/Agent/prepare_prompt/start/_99_snapshot_settings.py index 0d074c933c36b295117d8c08482154a1f9e25887..acc2e8b5f53a1a984e9bc9f8b509ea6dbdf3f8ed 100644 --- a/extensions/python/_functions/agent/Agent/prepare_prompt/start/_99_snapshot_settings.py +++ b/extensions/python/_functions/agent/Agent/prepare_prompt/start/_99_snapshot_settings.py @@ -1,3 +1,11 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:8f60a19ebaf9a35b14e788a31e3fce5a7285fa9fee1228374f2b25edff31a477 -size 328 +from helpers import settings +from helpers.extension import Extension + + +_TOKEN_KEY = "_prompt_settings_snapshot_token" + + +class SnapshotPromptSettings(Extension): + def execute(self, data: dict | None = None, **kwargs): + if isinstance(data, dict): + data[_TOKEN_KEY] = settings.begin_prompt_settings_snapshot() diff --git a/extensions/python/_functions/agent/AgentContext/run_task/end/_10_mark_state_dirty.py b/extensions/python/_functions/agent/AgentContext/run_task/end/_10_mark_state_dirty.py index 483bb4d7993c5584988422c5d3d07fc0a1417b81..be7afe395f0ec5c4b5902948e4c4879d1f19e70a 100644 --- a/extensions/python/_functions/agent/AgentContext/run_task/end/_10_mark_state_dirty.py +++ b/extensions/python/_functions/agent/AgentContext/run_task/end/_10_mark_state_dirty.py @@ -1,3 +1,14 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:02112c06ff56b8de8ed877895b58c0e0ed8057800e55eccf12a9e128f3878d4c -size 534 +from helpers.defer import DeferredTask +from helpers.extension import Extension +from helpers.state_monitor_integration import mark_dirty_all + + +class MarkStateDirty(Extension): + def execute(self, data: dict | None = None, **kwargs) -> None: + task = data.get("result") if isinstance(data, dict) else None + if isinstance(task, DeferredTask): + task.add_done_callback( + lambda _future: mark_dirty_all( + reason="agent.AgentContext.run_task_done", + ) + ) diff --git a/extensions/python/agent_init/AGENTS.md b/extensions/python/agent_init/AGENTS.md index c6808a4421044615013881734a823e77ee403a1a..3121c8244d7976da6fac1fb4a4cc808c3bcd15e5 100644 --- a/extensions/python/agent_init/AGENTS.md +++ b/extensions/python/agent_init/AGENTS.md @@ -1,3 +1,26 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:a0d9eb62f086092e47acc26b8d95af4020a180c2639da41a45f31987b76f6401 -size 625 +# Agent Init Extensions DOX + +## Purpose + +- Own backend extensions that run when an agent context initializes. + +## Ownership + +- Ordered Python files own initial UI message setup and profile settings load behavior. + +## Local Contracts + +- Keep initialization idempotent for contexts that may be restored or reloaded. +- Preserve ordering between initial message creation and profile settings loading. + +## Work Guidance + +- Coordinate changes with profile loading, settings resolution, and startup smoke checks. + +## Verification + +- Smoke-test new chat/context initialization after changes. + +## Child DOX Index + +No child DOX files. diff --git a/extensions/python/agent_init/_10_initial_message.py b/extensions/python/agent_init/_10_initial_message.py index ca0a6aa9e39b1d256d528f16ea6cec2444cd9621..cca5f798e36b5acb18436a4048d4f519ad61ed6f 100644 --- a/extensions/python/agent_init/_10_initial_message.py +++ b/extensions/python/agent_init/_10_initial_message.py @@ -1,3 +1,44 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:a600f43500b56a633d3dd0f8eec64e84fd32bfc6a43b55657652c8dc24ccc7a4 -size 1525 +import json +from agent import LoopData +from helpers.extension import Extension + + +class InitialMessage(Extension): + + def execute(self, **kwargs): + """ + Add an initial greeting message when first user message is processed. + Called only once per session via _process_chain method. + """ + if not self.agent: + return + + # Only add initial message for main agent (A0), not subordinate agents + if self.agent.number != 0: + return + + # If the context already contains log messages, do not add another initial message + if self.agent.context.log.logs: + return + + # Construct the initial message from prompt template + initial_message = self.agent.read_prompt("fw.initial_message.md") + + # add initial loop data to agent (for hist_add_ai_response) + self.agent.loop_data = LoopData(user_message=None) + + # Add the message to history as an AI response + msg = self.agent.hist_add_ai_response(initial_message) + + # json parse the message, get the tool_args text + initial_message_json = json.loads(initial_message) + initial_message_text = initial_message_json.get("tool_args", {}).get("text", "Hello! How can I help you?") + + # Add to log (green bubble) for immediate UI display + self.agent.context.log.log( + type="response", + content=initial_message_text, + finished=True, + update_progress="none", + id=msg.id, + ) diff --git a/extensions/python/agent_init/_15_load_profile_settings.py b/extensions/python/agent_init/_15_load_profile_settings.py index 6b6eb7793bd6ec76ec90e57c604596a56a6b53c0..a03913b5f9d6c280ea8d0a57544527e2c0539242 100644 --- a/extensions/python/agent_init/_15_load_profile_settings.py +++ b/extensions/python/agent_init/_15_load_profile_settings.py @@ -1,3 +1,53 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:661afc11f43f87eb7afcb2cef1690856bb298bde0e99945739fdc4132722d734 -size 2200 +from initialize import initialize_agent +from helpers import dirty_json, files, subagents, projects +from helpers.extension import Extension + + +class LoadProfileSettings(Extension): + + def execute(self, **kwargs) -> None: + + if not self.agent or not self.agent.config.profile: + return + + config_files = subagents.get_paths(self.agent, "settings.json", include_default=False, include_user=False) + settings_override = {} + for settings_path in config_files: + if files.exists(settings_path): + try: + override_settings_str = files.read_file(settings_path) + override_settings = dirty_json.try_parse(override_settings_str) + if isinstance(override_settings, dict): + settings_override.update(override_settings) + else: + raise Exception( + f"Subordinate settings in {settings_path} must be a JSON object." + ) + except Exception as e: + self.agent.context.log.log( + type="error", + content=( + f"Error loading subordinate settings from {settings_path} for " + f"profile '{self.agent.config.profile}': {e}" + ), + ) + + if settings_override: + current_config = self.agent.config + new_config = initialize_agent(override_settings=settings_override) + + for override_key, config_attr in ( + ("agent_profile", "profile"), + ("mcp_servers", "mcp_servers"), + ): + if override_key not in settings_override: + setattr(new_config, config_attr, getattr(current_config, config_attr)) + self.agent.config = new_config + # self.agent.context.log.log( + # type="info", + # content=( + # "Loaded custom settings for agent " + # f"{self.agent.number} with profile '{self.agent.config.profile}'." + # ), + # ) + diff --git a/extensions/python/banners/AGENTS.md b/extensions/python/banners/AGENTS.md index e1aa0f4fe295f9f7d51f4d2c6a21b84cba5b8514..f2ce20f10f4d5799d1edca5562e58f8af639da69 100644 --- a/extensions/python/banners/AGENTS.md +++ b/extensions/python/banners/AGENTS.md @@ -1,3 +1,28 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:9384eec1a33ca8900836671c77730e4c3e7ef573abe31700150c8f7b6e2950ff -size 808 +# Banner Extensions DOX + +## Purpose + +- Own backend banner and discovery-card contributions. + +## Ownership + +- Ordered Python files append alert banners or discovery cards to the mutable `banners` list. + +## Local Contracts + +- Banner IDs must be unique and stable. +- Use supported banner/card fields and types only. +- Banner HTML links that trigger WebUI behavior should use supported structured actions, such as `data-banner-action`, instead of inline JavaScript handlers. +- Do not expose secrets, local paths, or raw system diagnostics in banner text. + +## Work Guidance + +- Gate setup or warning banners on current configuration/status where possible. + +## Verification + +- Smoke-test welcome-screen banner rendering, ordering, dismissal, and CTA behavior after changes. + +## Child DOX Index + +No child DOX files. diff --git a/extensions/python/banners/_10_unsecured_connection.py b/extensions/python/banners/_10_unsecured_connection.py index 9ed757b1c2d8582d547b4a367cdaf7744dfed954..19145fef3f7ebfdee434fa8404bd8ac34b3978a2 100644 --- a/extensions/python/banners/_10_unsecured_connection.py +++ b/extensions/python/banners/_10_unsecured_connection.py @@ -1,3 +1,63 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:6a8d4f8ed25d1e17d786ae5ea6b1f1d6d29f2b80bc8cce4d4bb4807c5d56c623 -size 2640 +from helpers.extension import Extension +from helpers import dotenv +import re + + +class UnsecuredConnectionCheck(Extension): + """Check: non-local without credentials, or credentials over non-HTTPS.""" + + async def execute(self, banners: list = [], frontend_context: dict = {}, **kwargs): + hostname = frontend_context.get("hostname", "") + protocol = frontend_context.get("protocol", "") + + auth_login = dotenv.get_dotenv_value(dotenv.KEY_AUTH_LOGIN, "") + auth_password = dotenv.get_dotenv_value(dotenv.KEY_AUTH_PASSWORD, "") + has_credentials = bool(auth_login and auth_login.strip() and auth_password and auth_password.strip()) + + is_local = self._is_localhost(hostname) + is_https = protocol == "https:" + + if not is_local and not has_credentials: + banners.append({ + "id": "unsecured-connection", + "type": "warning", + "priority": 80, + "title": "Unsecured Connection", + "html": """You are accessing Agent Zero from a non-local address without authentication. + + Configure credentials in Settings → External Services → Authentication.""", + "dismissible": True, + "source": "backend" + }) + + if has_credentials and not is_local and not is_https: + banners.append({ + "id": "credentials-unencrypted", + "type": "warning", + "priority": 90, + "title": "Credentials May Be Sent Unencrypted", + "html": """Your connection is not using HTTPS. Login credentials may be transmitted in plain text. + Consider using HTTPS or a secure tunnel.""", + "dismissible": True, + "source": "backend" + }) + + def _is_localhost(self, hostname: str) -> bool: + local_patterns = ["localhost", "127.0.0.1", "::1", "0.0.0.0"] + + if hostname in local_patterns: + return True + + # RFC1918 private ranges + if re.match(r"^192\.168\.\d{1,3}\.\d{1,3}$", hostname): + return True + if re.match(r"^10\.\d{1,3}\.\d{1,3}\.\d{1,3}$", hostname): + return True + if re.match(r"^172\.(1[6-9]|2\d|3[01])\.\d{1,3}\.\d{1,3}$", hostname): + return True + + # .local domains + if hostname.endswith(".local"): + return True + + return False diff --git a/extensions/python/banners/_30_system_resources.py b/extensions/python/banners/_30_system_resources.py index 899b15ca93eb52e54ef4ee493c5b19106ac52870..3c1949f4f422b28980c27bb83f9cdf4e4380c918 100644 --- a/extensions/python/banners/_30_system_resources.py +++ b/extensions/python/banners/_30_system_resources.py @@ -1,3 +1,151 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:c25cebb701f5ee4166d857d7bd34ea1d149507108691cd94cc78c553a606ec3d -size 5422 +from helpers.extension import Extension +import os +import psutil + + +class SystemResourcesCheck(Extension): + async def execute(self, banners: list = [], frontend_context: dict = {}, **kwargs): + try: + cpu_percent = psutil.cpu_percent(interval=0.1) + except Exception: + cpu_percent = None + + try: + cpu_cores = psutil.cpu_count(logical=True) + except Exception: + cpu_cores = None + + load_avg = self._get_load_average() + + try: + vm = psutil.virtual_memory() + ram_percent = vm.percent + ram_used_gb = (vm.total - vm.available) / (1024 ** 3) + ram_total_gb = vm.total / (1024 ** 3) + except Exception: + ram_percent = None + + ram_used_gb = None + ram_total_gb = None + + disk_percent, disk_used_gb, disk_total_gb, disk_path = self._get_disk_usage() + + try: + net = psutil.net_io_counters() + net_sent = self._format_bytes(net.bytes_sent) + net_recv = self._format_bytes(net.bytes_recv) + except Exception: + net_sent = "N/A" + net_recv = "N/A" + + load_value = "N/A" + if load_avg: + la1, la5, la15 = load_avg + load_value = f"{la1:.2f} / {la5:.2f} / {la15:.2f}" + + if disk_percent is None or disk_used_gb is None or disk_total_gb is None: + disk_value = "N/A" + else: + disk_value = f"{disk_used_gb:.2f}/{disk_total_gb:.2f} GB" + + if cpu_percent is None: + cpu_value = "N/A" + else: + cores_value = "" if cpu_cores is None else f" ({cpu_cores} cores)" + cpu_value = f"{cpu_percent:.0f}%{cores_value}" + + if ram_percent is None or ram_used_gb is None or ram_total_gb is None: + ram_value = "N/A" + else: + ram_value = f"{ram_used_gb:.2f}/{ram_total_gb:.2f} GB" + + cpu_bar = self._bar_html(cpu_percent) + ram_bar = self._bar_html(ram_percent) + disk_bar = self._bar_html(disk_percent) + + banners.append({ + "id": "system-resources", + "type": "info", + "priority": 10, + "title": "System Resources", + "html": ( + "
" + "
" + "
" + "
CPU
" + f"
{cpu_value}
" + "
" + f"{cpu_bar}" + "
" + "
" + "
" + "
RAM
" + f"
{ram_value}
" + "
" + f"{ram_bar}" + "
" + "
" + "
" + "
Disk
" + f"
{disk_value}
" + "
" + f"{disk_bar}" + "
" + "
" + "
" + "
Load (1/5/15)
" + f"
{load_value}
" + "
" + "
" + "
Net (since boot)
" + f"
{net_sent} sent / {net_recv} recv
" + "
" + "
" + "
" + ), + "dismissible": True, + "source": "backend", + }) + + def _bar_html(self, percent: float | None) -> str: + if percent is None: + return "" + + p = max(0.0, min(100.0, float(percent))) + if p >= 85: + color = "#ef4444" + elif p >= 70: + color = "#f59e0b" + else: + color = "#22c55e" + + return ( + "
" + f"" + "
" + ) + + def _get_load_average(self) -> tuple[float, float, float] | None: + try: + return os.getloadavg() + except Exception: + return None + + def _get_disk_usage(self) -> tuple[float | None, float | None, float | None, str]: + for path in ["/", os.path.expanduser("~")]: + try: + usage = psutil.disk_usage(path) + used_gb = usage.used / (1024 ** 3) + total_gb = usage.total / (1024 ** 3) + return usage.percent, used_gb, total_gb, path + except Exception: + continue + return None, None, None, "/" + + def _format_bytes(self, value: int) -> str: + size = float(value) + for unit in ["B", "KB", "MB", "GB", "TB", "PB"]: + if size < 1024: + return f"{size:.1f} {unit}" + size /= 1024 + return f"{size:.1f} EB" diff --git a/extensions/python/before_main_llm_call/AGENTS.md b/extensions/python/before_main_llm_call/AGENTS.md index 10863d53a77dadd4a7d8254a01bb5b0c5d356f0f..36214e3b3f6b93f1fa0b18bd97f0a0ec27c9eee3 100644 --- a/extensions/python/before_main_llm_call/AGENTS.md +++ b/extensions/python/before_main_llm_call/AGENTS.md @@ -1,3 +1,26 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:9a6e854a5cf0d094abc2bc4da91833b82b0301d173ddff8537b16b341a60a868 -size 655 +# Before Main LLM Call Extensions DOX + +## Purpose + +- Own backend behavior that runs immediately before the main LLM call. + +## Ownership + +- Ordered Python files own pre-call logging or context preparation for streaming/model execution. + +## Local Contracts + +- Do not mutate prompt or history data unless the hook contract explicitly passes mutable state for that purpose. +- Avoid logging secrets, hidden prompt sections, or private user data. + +## Work Guidance + +- Keep this hook light because it sits on the main model hot path. + +## Verification + +- Run targeted model-call or streaming tests after behavior changes. + +## Child DOX Index + +No child DOX files. diff --git a/extensions/python/before_main_llm_call/_10_log_for_stream.py b/extensions/python/before_main_llm_call/_10_log_for_stream.py index 04903bdd49063a53c644e7cb182ebcfa9938cd93..66fab40112d6e884f5d49ae860e7154fac0d98c8 100644 --- a/extensions/python/before_main_llm_call/_10_log_for_stream.py +++ b/extensions/python/before_main_llm_call/_10_log_for_stream.py @@ -1,3 +1,33 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:39f2b897c81318b9f399bd96aa5883bfa866bee8857a62cdd69061aacfed94ac -size 1107 +from helpers import persist_chat, tokens +from helpers.extension import Extension +from agent import LoopData +import asyncio +from helpers.log import LogItem +from helpers import log +import math +import uuid + + +class LogForStream(Extension): + + async def execute(self, loop_data: LoopData = LoopData(), text: str = "", **kwargs): + if not self.agent: + return + + # create log message and store it in loop data temporary params + if "log_item_generating" not in loop_data.params_temporary: + loop_data.params_temporary["log_item_generating"] = ( + self.agent.context.log.log( + type="agent", + heading=build_default_heading(self.agent), + id=str(uuid.uuid4()), + ) + ) + +def build_heading(agent, text: str, icon: str = "network_intelligence"): + # Include agent identifier for all agents (A0:, A1:, A2:, etc.) + agent_prefix = f"{agent.agent_name}: " + return f"{agent_prefix}{text}" + +def build_default_heading(agent): + return build_heading(agent, "Calling LLM...") \ No newline at end of file diff --git a/extensions/python/error_format/AGENTS.md b/extensions/python/error_format/AGENTS.md index a11b09ee38dfe20bb6a0d44a6652bc3400b912fe..6dd136a737ed62fefcea000bfc8b401ab30f41d2 100644 --- a/extensions/python/error_format/AGENTS.md +++ b/extensions/python/error_format/AGENTS.md @@ -1,3 +1,26 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:9b7b73348ce16977cdbdcb128c99408b750a985698adf49d8c4a8c2976c2b059 -size 616 +# Error Format Extensions DOX + +## Purpose + +- Own backend error formatting and masking before errors are shown or logged. + +## Ownership + +- Ordered Python files own mutation of error-format data passed by the hook. + +## Local Contracts + +- Preserve secret masking and safe user-facing error messages. +- Do not expose raw tokens, credentials, hidden prompts, or private payloads. + +## Work Guidance + +- Keep masking rules conservative and synchronized with tool/model error surfaces. + +## Verification + +- Test or inspect representative masked and unmasked error paths after changes. + +## Child DOX Index + +No child DOX files. diff --git a/extensions/python/error_format/_10_mask_errors.py b/extensions/python/error_format/_10_mask_errors.py index cdfd1575d863e5e6e945d85cfd11c73bdf5837ef..3547e52208ddee8453ba4d75a68f12f0652a026f 100644 --- a/extensions/python/error_format/_10_mask_errors.py +++ b/extensions/python/error_format/_10_mask_errors.py @@ -1,3 +1,20 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:30811f67fa20158348d9bd71dc8ad3c11c5562330e090d9fcae9ef2e34d9dd18 -size 523 +from helpers.extension import Extension +from helpers.secrets import get_secrets_manager + + +class MaskErrorSecrets(Extension): + + async def execute(self, **kwargs): + if not self.agent: + return + + # Get error data from kwargs + msg = kwargs.get("msg") + if not msg: + return + + secrets_mgr = get_secrets_manager(self.agent.context) + + # Mask the error message + if "message" in msg: + msg["message"] = secrets_mgr.mask_values(msg["message"]) diff --git a/extensions/python/hist_add_before/AGENTS.md b/extensions/python/hist_add_before/AGENTS.md index 9b39114d1c74bd30dcf2e6c097c0f14e004c7e5b..791b47294aa8c747bee7d31a007c515ea4a4f627 100644 --- a/extensions/python/hist_add_before/AGENTS.md +++ b/extensions/python/hist_add_before/AGENTS.md @@ -1,3 +1,26 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:5857c34d73e867c3034d4a2edad9d2e054469e9b2a81b5ee843fcd8ee8547e23 -size 629 +# History Add Before Extensions DOX + +## Purpose + +- Own preprocessing before messages are added to agent history. + +## Ownership + +- Ordered Python files own history content masking before persistence or model reuse. + +## Local Contracts + +- Preserve secret and sensitive-content masking before history storage. +- Do not remove fields required by downstream history organization or replay. + +## Work Guidance + +- Coordinate history mutation changes with message persistence and plugin-owned history consumers. + +## Verification + +- Test message history insertion for masked content after changes. + +## Child DOX Index + +No child DOX files. diff --git a/extensions/python/hist_add_before/_10_mask_content.py b/extensions/python/hist_add_before/_10_mask_content.py index b9defa010eef28908f73da019535b6565b124f68..5b09e14c542982496dae859e6c5d1326d53f98e9 100644 --- a/extensions/python/hist_add_before/_10_mask_content.py +++ b/extensions/python/hist_add_before/_10_mask_content.py @@ -1,3 +1,35 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:eb613a29492b4e9fa9b238de709d3fcee747b7ab6dee78e7871b0371dafb3f41 -size 1212 +from helpers.extension import Extension +from helpers.secrets import get_secrets_manager + + +class MaskHistoryContent(Extension): + + def execute(self, **kwargs): + if not self.agent: + return + + # Get content data from kwargs + content_data = kwargs.get("content_data") + if not content_data: + return + + try: + secrets_mgr = get_secrets_manager(self.agent.context) + + # Mask the content before adding to history + content_data["content"] = self._mask_content(content_data["content"], secrets_mgr) + except Exception as e: + # If masking fails, proceed without masking + pass + + def _mask_content(self, content, secrets_mgr): + """Recursively mask secrets in message content.""" + if isinstance(content, str): + return secrets_mgr.mask_values(content) + elif isinstance(content, list): + return [self._mask_content(item, secrets_mgr) for item in content] + elif isinstance(content, dict): + return {k: self._mask_content(v, secrets_mgr) for k, v in content.items()} + else: + # For other types, return as-is + return content diff --git a/extensions/python/hist_add_tool_result/AGENTS.md b/extensions/python/hist_add_tool_result/AGENTS.md index 73ab089c7831291667e7732d4fc9bdeb08dba30f..c1dd0ecf284e34a844642e1ba035aa2416b5c21d 100644 --- a/extensions/python/hist_add_tool_result/AGENTS.md +++ b/extensions/python/hist_add_tool_result/AGENTS.md @@ -1,3 +1,27 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:b7fa7f5240203dec28a2b603a075b29bcbdaeecaf77450cb1dcf5f410577ffc7 -size 692 +# History Tool Result Extensions DOX + +## Purpose + +- Own processing after tool results are added to history. + +## Ownership + +- Ordered Python files own tool-call file persistence and related history side effects. + +## Local Contracts + +- Preserve tool result traceability without leaking secrets. +- Keep file artifacts inside expected runtime/user-owned paths. +- Skip BACKGROUND contexts; background workers must remain ephemeral and must not create chat message files. + +## Work Guidance + +- Coordinate changes with tool output storage and chat persistence behavior. + +## Verification + +- Smoke-test a tool call that produces persisted output after changes. + +## Child DOX Index + +No child DOX files. diff --git a/extensions/python/hist_add_tool_result/_90_save_tool_call_file.py b/extensions/python/hist_add_tool_result/_90_save_tool_call_file.py index f5d855288bb894ed7ecc6be0c2ea94e05d901d98..fd5a328546559de92ae866b37bb6ac68d5b3e927 100644 --- a/extensions/python/hist_add_tool_result/_90_save_tool_call_file.py +++ b/extensions/python/hist_add_tool_result/_90_save_tool_call_file.py @@ -1,3 +1,44 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:8dd610d93ebf73e70c78f2eb476b1c766dae1ecf330f4597b894064a21e03874 -size 1208 +from typing import Any +from agent import AgentContextType +from helpers.extension import Extension +from helpers import files, persist_chat +import os, re + +LEN_MIN = 500 + +class SaveToolCallFile(Extension): + def execute(self, data: dict[str, Any] | None = None, **kwargs): + if not self.agent: + return + + if self.agent.context.type == AgentContextType.BACKGROUND: + return + + if not data: + return + + # get tool call result + result = data.get("tool_result") if isinstance(data, dict) else None + if result is None: + return + + # skip short results + if len(str(result)) < LEN_MIN: + return + + # message files directory + msgs_folder = persist_chat.get_chat_msg_files_folder(self.agent.context.id) + os.makedirs(msgs_folder, exist_ok=True) + + # count the files in the directory + last_num = len(os.listdir(msgs_folder)) + + # create new file + new_file = files.get_abs_path(msgs_folder, f"{last_num+1}.txt") + files.write_file( + new_file, + result, + ) + + # add the path to the history + data["file"] = new_file diff --git a/extensions/python/job_loop/AGENTS.md b/extensions/python/job_loop/AGENTS.md index 09ab88dbac4252b44171783a4a3e4ad6d3e31ad4..afa7597a3dca5282d0f2b7ee92b492360d2bdab2 100644 --- a/extensions/python/job_loop/AGENTS.md +++ b/extensions/python/job_loop/AGENTS.md @@ -1,3 +1,26 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:a6af0f3ca610f74f000248f7483e06b217b803155116f24fd818137c87ae8eb4 -size 600 +# Job Loop Extensions DOX + +## Purpose + +- Own periodic backend maintenance jobs. + +## Ownership + +- Ordered Python files own cleanup of expired API chats, cache trimming, and future job-loop tasks. + +## Local Contracts + +- Jobs must be idempotent and safe to run repeatedly. +- Keep cleanup scoped to owned caches, temporary contexts, or documented runtime state. + +## Work Guidance + +- Avoid expensive work on every loop; use timestamps or thresholds when practical. + +## Verification + +- Run targeted cleanup/cache tests or smoke-test job-loop startup after changes. + +## Child DOX Index + +No child DOX files. diff --git a/extensions/python/job_loop/_20_cleanup_expired_api_chats.py b/extensions/python/job_loop/_20_cleanup_expired_api_chats.py index 76593d5b4024cd920a50dd7c747cf810ac845700..b8d9da0a096deba5af16463fbf49939a32710c02 100644 --- a/extensions/python/job_loop/_20_cleanup_expired_api_chats.py +++ b/extensions/python/job_loop/_20_cleanup_expired_api_chats.py @@ -1,3 +1,63 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:77ff57423672bedcbdf501f27cbcb7816192e088e2e98893be9c76ae819e67a9 -size 2119 +from datetime import datetime, timedelta, timezone +from typing import Any + +from agent import AgentContext +from helpers import persist_chat +from helpers.extension import Extension +from helpers.print_style import PrintStyle +from helpers.state_monitor_integration import mark_dirty_all + + +CHECK_INTERVAL = timedelta(hours=1) +LIFETIME_KEY = "lifetime_hours" + + +class CleanupExpiredApiChats(Extension): + _last_check: datetime | None = None + + async def execute(self, data: dict[str, Any] | None = None, **kwargs): + now = datetime.now(timezone.utc) + if type(self)._last_check and now - type(self)._last_check < CHECK_INTERVAL: + return + type(self)._last_check = now + + removed = 0 + for context in list(AgentContext.all()): + lifetime_hours = context.get_data(LIFETIME_KEY) + if lifetime_hours is None: + continue + + try: + lifetime = timedelta(hours=float(lifetime_hours)) + except (TypeError, ValueError): + PrintStyle.error( + f"Invalid chat lifetime for {context.id}: {lifetime_hours}" + ) + continue + + if lifetime <= timedelta(0) or context.is_running(): + continue + + last_message = _as_utc(context.last_message) + if now - last_message <= lifetime: + continue + + try: + context.reset() + AgentContext.remove(context.id) + persist_chat.remove_chat(context.id) + removed += 1 + PrintStyle().print(f"Cleaned up expired API chat: {context.id}") + except Exception as e: + PrintStyle.error(f"Failed to cleanup expired API chat {context.id}: {e}") + + if removed: + mark_dirty_all(reason="job_loop.CleanupExpiredApiChats") + + +def _as_utc(value: datetime | None) -> datetime: + if value is None: + return datetime.fromtimestamp(0, timezone.utc) + if value.tzinfo is None: + return value.replace(tzinfo=timezone.utc) + return value.astimezone(timezone.utc) diff --git a/extensions/python/job_loop/_50_trim_cache.py b/extensions/python/job_loop/_50_trim_cache.py index fec29e9519f394915733af9137f5cc5de80c59c4..74989fa86351910ad23a216d49904bf8c064b8ea 100644 --- a/extensions/python/job_loop/_50_trim_cache.py +++ b/extensions/python/job_loop/_50_trim_cache.py @@ -1,3 +1,9 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:a5b536371abc86345790f9014baa645fc7d33fcd0a1062dc0fc27c56b7948587 -size 274 +from typing import Any +from helpers.extension import Extension +from helpers import cache + + +class SaveToolCallFile(Extension): + def execute(self, data: dict[str, Any] | None = None, **kwargs): + # trim unused cache entries + cache.trim_cache("*", seconds=300) diff --git a/extensions/python/message_loop_end/AGENTS.md b/extensions/python/message_loop_end/AGENTS.md index 873abd0829e2cfed53768e937d2d5b21d6051d8a..2200c4aa3b773ca996e05bb2a538dbbe4012cb8c 100644 --- a/extensions/python/message_loop_end/AGENTS.md +++ b/extensions/python/message_loop_end/AGENTS.md @@ -1,3 +1,27 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:aad03752d410b9e80eef91fa1e1b5a45b7907f2085001f6591e47878495d6f5c -size 803 +# Message Loop End Extensions DOX + +## Purpose + +- Own backend behavior that runs after a message loop completes. + +## Ownership + +- Ordered Python files own history organization and chat persistence at loop end. + +## Local Contracts + +- Preserve history consistency before saving chats. +- Do not skip persistence for successful loops unless the hook contract explicitly permits it. +- History compression that rewrites local history must clear the active Responses provider continuation while preserving stored response IDs for cleanup. + +## Work Guidance + +- Coordinate changes with chat serialization, history organization, and WebUI refresh behavior. + +## Verification + +- Smoke-test sending a message, reloading the chat, and checking persisted history after changes. + +## Child DOX Index + +No child DOX files. diff --git a/extensions/python/message_loop_end/_10_organize_history.py b/extensions/python/message_loop_end/_10_organize_history.py index e7b5130d3b849ce08ed7441c4aec81432392b70a..0cc205bd6ba959ebcf25f4926359e60ed7678fdf 100644 --- a/extensions/python/message_loop_end/_10_organize_history.py +++ b/extensions/python/message_loop_end/_10_organize_history.py @@ -1,3 +1,30 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:70998648652bb1192f47f3bc4a4907a1b7860045bf6f76dbf075841cea0394bb -size 1047 +from helpers.extension import Extension +from agent import LoopData +from helpers.defer import DeferredTask, THREAD_BACKGROUND +from helpers.history import clear_responses_provider_state + +DATA_NAME_TASK = "_organize_history_task" + + +async def compress_history(agent) -> bool: + compressed = bool(await agent.history.compress()) + if compressed: + clear_responses_provider_state(agent) + return compressed + + +class OrganizeHistory(Extension): + async def execute(self, loop_data: LoopData = LoopData(), **kwargs): + if not self.agent: + return + + # is there a running task? if yes, skip this round, the wait extension will double check the context size + task: DeferredTask|None = self.agent.get_data(DATA_NAME_TASK) + if task and not task.is_ready(): + return + + # start task + task = DeferredTask(thread_name=THREAD_BACKGROUND) + task.start_task(compress_history, self.agent) + # set to agent to be able to wait for it + self.agent.set_data(DATA_NAME_TASK, task) diff --git a/extensions/python/message_loop_end/_90_save_chat.py b/extensions/python/message_loop_end/_90_save_chat.py index 6d6601e4c052aebeef85a66adac51dc54b29ca86..72a3410be9e2ec1a7188214bb8d7bea3fae3f6af 100644 --- a/extensions/python/message_loop_end/_90_save_chat.py +++ b/extensions/python/message_loop_end/_90_save_chat.py @@ -1,3 +1,15 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:8f2efeb0b9ddafa494eb58cdc1f649eb2edf9f0ae7c16d8f58d6cb95d6bc87ab -size 479 +from helpers.extension import Extension +from agent import LoopData, AgentContextType +from helpers import persist_chat + + +class SaveChat(Extension): + async def execute(self, loop_data: LoopData = LoopData(), **kwargs): + if not self.agent: + return + + # Skip saving BACKGROUND contexts as they should be ephemeral + if self.agent.context.type == AgentContextType.BACKGROUND: + return + + persist_chat.save_tmp_chat(self.agent.context) diff --git a/extensions/python/message_loop_prompts_after/AGENTS.md b/extensions/python/message_loop_prompts_after/AGENTS.md index fdf7d914f574d9048913b5ac5f2c093faaf672b8..1a183d994789723ae818035b0204362f166807b3 100644 --- a/extensions/python/message_loop_prompts_after/AGENTS.md +++ b/extensions/python/message_loop_prompts_after/AGENTS.md @@ -1,3 +1,33 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:42e777c4abc98d45ef762257a7bd55f079df91a7b6dc1ef1bffbfc72b288cf4d -size 1383 +# Message Loop Prompts After Extensions DOX + +## Purpose + +- Own prompt protocol, prompt extras, and history reattachment around primary message-loop prompt construction. + +## Ownership + +- Ordered Python files own current datetime, relevant-skill hints, loaded-skill history reattachment, agent info, parallel job status, and workdir extras injection. +- Explicitly loaded skill bodies belong in tool-result history with metadata so they can survive persistence and be reattached after compaction. +- Explicitly loaded skill IDs are chat-wide context data, not agent-local state. +- Skills must not write selected or loaded skill bodies into protocol or extras. + +## Local Contracts + +- Keep injected content bounded and clearly attributed. +- Preserve ordering where later prompt extras depend on earlier recall or load results. +- Do not expose secrets or private files from workdir extras. +- Relevant-skill recall should search the raw user message when available, not the rendered history wrapper. +- Relevant-skill hints must not advertise loading when profile policy blocks + `skills_tool`. + +## Work Guidance + +- Coordinate prompt protocol, history-reattachment, and prompt-extra changes with skill, workdir, and profile contracts. + +## Verification + +- Inspect rendered prompt protocol/history/extras or run prompt-construction tests after changes. + +## Child DOX Index + +No child DOX files. diff --git a/extensions/python/message_loop_prompts_after/_60_include_current_datetime.py b/extensions/python/message_loop_prompts_after/_60_include_current_datetime.py index faef383960337c1080bdbf4e2cdaf1f96f95f94a..c57c52291f745046b9e38add33ce18749bb3f60a 100644 --- a/extensions/python/message_loop_prompts_after/_60_include_current_datetime.py +++ b/extensions/python/message_loop_prompts_after/_60_include_current_datetime.py @@ -1,3 +1,19 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:6737327bbd60eb39b4cf9d581c243a472cba4f69e2abe6bb42f207970d3f601e -size 633 +from helpers.extension import Extension +from agent import LoopData +from helpers.localization import Localization + + +class IncludeCurrentDatetime(Extension): + async def execute(self, loop_data: LoopData = LoopData(), **kwargs): + if not self.agent: + return + + current_datetime = Localization.get().now().strftime("%Y-%m-%d %H:%M:%S %Z") + + # read prompt + datetime_prompt = self.agent.read_prompt( + "agent.system.datetime.md", date_time=current_datetime + ) + + # add current datetime to the loop data + loop_data.extras_temporary["current_datetime"] = datetime_prompt diff --git a/extensions/python/message_loop_prompts_after/_63_recall_relevant_skills.py b/extensions/python/message_loop_prompts_after/_63_recall_relevant_skills.py index 39dd620c03a6d9a3ef23abfece5bea7919f09519..ccfd6f42cf213bf35ad5f65f95e3ea1c7619f937 100644 --- a/extensions/python/message_loop_prompts_after/_63_recall_relevant_skills.py +++ b/extensions/python/message_loop_prompts_after/_63_recall_relevant_skills.py @@ -1,3 +1,45 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:130e5e731e029d363cf3cf361b33613c846a40627a81fb0eb2967c568fff24ea -size 1544 +from agent import LoopData +from helpers.extension import Extension +from helpers import skills as skills_helper, tool_policy + + +class RecallRelevantSkills(Extension): + async def execute(self, loop_data: LoopData = LoopData(), **kwargs): + if not self.agent or loop_data.iteration != 0: + return + if not tool_policy.resolve_tool(self.agent, "skills_tool").allowed: + return + + content = loop_data.user_message.content if loop_data.user_message else "" + if isinstance(content, dict): + user_instruction = str(content.get("user_message") or "").strip() + else: + user_instruction = ( + loop_data.user_message.output_text() if loop_data.user_message else "" + ).strip() + if len(user_instruction) < 8: + return + + matches = skills_helper.search_skills( + user_instruction, + limit=6, + agent=self.agent, + ) + if not matches: + return + + lines: list[str] = [] + for skill in matches: + name = skill.name.strip().replace("\n", " ")[:100] + desc = (skill.description or "").replace("\n", " ").strip() + if len(desc) > 220: + desc = desc[:220].rstrip() + "…" + lines.append(f"- {name}: {desc}") + + if not lines: + return + + loop_data.extras_temporary["relevant_skills"] = self.agent.read_prompt( + "agent.system.skills.relevant.md", + skills="\n".join(lines), + ) diff --git a/extensions/python/message_loop_prompts_after/_65_include_loaded_skills.py b/extensions/python/message_loop_prompts_after/_65_include_loaded_skills.py index 20fc4410f42cf8d78e33a28051a9915e919a30b2..6c604005a065727eb1fa019a2bd178ac027852b9 100644 --- a/extensions/python/message_loop_prompts_after/_65_include_loaded_skills.py +++ b/extensions/python/message_loop_prompts_after/_65_include_loaded_skills.py @@ -1,3 +1,78 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:3d911949e4103bf7d9a756a85517d4cca75aeab223d8f57956244fdf110a1cd3 -size 2668 +from helpers.extension import Extension +from helpers import skills, tokens +from agent import LoopData + + +SKILL_REATTACHMENT_TOKEN_BUDGET = 12_000 +SKILL_REATTACHMENT_HEADER = ( + "Reattached loaded skill instructions after history compaction." +) + + +class IncludeLoadedSkills(Extension): + async def execute(self, loop_data: LoopData = LoopData(), **kwargs): + if not self.agent: + return + + skill_names = skills.get_loaded_skill_names(self.agent) + if not skill_names: + return + + # Loaded skill bodies live in tool-result history. This hook only keeps + # the ledger clean and restores bodies that compaction hid. + visible_skill_names = [] + loaded_skills = [] + for skill_name in skill_names: + skill = skills.find_skill(skill_name, agent=self.agent) + if not skill: + continue + visible_skill_names.append(skill.name) + loaded_skills.append(skill) + skills.set_loaded_skill_names(self.agent, visible_skill_names) + + self._reattach_missing_skill_bodies(loop_data, loaded_skills) + + def _reattach_missing_skill_bodies(self, loop_data: LoopData, loaded_skills): + if not self.agent or not loaded_skills: + return + + visible_skill_names = _visible_skill_names(loop_data.history_output) + selected = [] + used_tokens = 0 + + for skill in reversed(loaded_skills): + if skill.name in visible_skill_names: + continue + + skill_data = skills.load_skill_for_agent( + skill_name=skill.name, + agent=self.agent, + ) + message = f"{SKILL_REATTACHMENT_HEADER}\n\n{skill_data}" + message_tokens = tokens.approximate_tokens(message) + if used_tokens + message_tokens > SKILL_REATTACHMENT_TOKEN_BUDGET: + continue + + selected.append((skill, message)) + used_tokens += message_tokens + + for skill, message in reversed(selected): + history_message = self.agent.hist_add_tool_result( + "skills_tool", + message, + skill_instructions={ + "name": skill.name, + "path": str(skill.path), + "source": "skills_tool:reattach", + "content_included": True, + }, + ) + loop_data.history_output.extend(history_message.output()) + + +def _visible_skill_names(history_output) -> set[str]: + return { + name + for message in history_output or [] + if (name := skills.skill_instruction_name(message)) + } diff --git a/extensions/python/message_loop_prompts_after/_70_include_agent_info.py b/extensions/python/message_loop_prompts_after/_70_include_agent_info.py index 0702cbd63723213dd0c75775be2da12cffebe98d..c4dc3f4d7c67c2ea6b91c318b3a46093631d7df4 100644 --- a/extensions/python/message_loop_prompts_after/_70_include_agent_info.py +++ b/extensions/python/message_loop_prompts_after/_70_include_agent_info.py @@ -1,3 +1,27 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:c10a045b764ddc51c2f00ba3e7a9cc702fd8acd3b31f3b7942707eeea1c432e9 -size 931 +from helpers.extension import Extension +from agent import LoopData + + +class IncludeAgentInfo(Extension): + async def execute(self, loop_data: LoopData = LoopData(), **kwargs): + if not self.agent: + return + + # read prompt + from plugins._model_config.helpers.model_config import ( + get_chat_model_config, + get_effective_preset_name, + ) + chat_cfg = get_chat_model_config(self.agent) + preset_name = get_effective_preset_name(self.agent) + + agent_info_prompt = self.agent.read_prompt( + "agent.extras.agent_info.md", + number=self.agent.number, + profile=self.agent.config.profile or "Default", + llm=chat_cfg.get("provider", "") + "/" + chat_cfg.get("name", ""), + preset=preset_name, + ) + + # add agent info to the prompt + loop_data.extras_temporary["agent_info"] = agent_info_prompt diff --git a/extensions/python/message_loop_prompts_after/_72_include_parallel_jobs.py b/extensions/python/message_loop_prompts_after/_72_include_parallel_jobs.py index 094d7a2bba7dbd87223873328010aae2a568815e..70bc0ce4eb9e5b9fe1fe90d98c3792da7cf8cb27 100644 --- a/extensions/python/message_loop_prompts_after/_72_include_parallel_jobs.py +++ b/extensions/python/message_loop_prompts_after/_72_include_parallel_jobs.py @@ -1,3 +1,13 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:2d4e5220acd60cb3de270e2e828db61c999c951ab5d9f9bbb9f5457bc1a15f2b -size 423 +from helpers.extension import Extension +from agent import LoopData +from helpers import parallel_tools + + +class IncludeParallelJobs(Extension): + async def execute(self, loop_data: LoopData = LoopData(), **kwargs): + if not self.agent: + return + + extras = await parallel_tools.build_parallel_jobs_extras(self.agent) + if extras: + loop_data.extras_temporary["parallel_jobs"] = extras diff --git a/extensions/python/message_loop_prompts_after/_75_include_workdir_extras.py b/extensions/python/message_loop_prompts_after/_75_include_workdir_extras.py index ab05be42311cb1835120f3ecf044efb0029cc060..a6b5cf10b83de9e0306252836ec72edb1465220d 100644 --- a/extensions/python/message_loop_prompts_after/_75_include_workdir_extras.py +++ b/extensions/python/message_loop_prompts_after/_75_include_workdir_extras.py @@ -1,3 +1,96 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:bc46c3d0e739e5f9f66e753ff11d589dd857eff3ad328d9b00244e65ab307dcd -size 3173 +from helpers.extension import Extension +from agent import LoopData +from helpers import projects +from helpers import settings +from helpers import runtime +from helpers import file_tree +from helpers import files + +class IncludeWorkdirExtras(Extension): + async def execute(self, loop_data: LoopData = LoopData(), **kwargs): + if not self.agent: + return + + project_name = projects.get_context_project_name(self.agent.context) + + enabled = False + max_depth = 0 + max_files = 0 + max_folders = 0 + max_lines = 0 + gitignore_raw = "" + folder = "" + file_structure = "" + + if project_name: + project = projects.load_basic_project_data(project_name) + enabled = project["file_structure"]["enabled"] + + if not enabled: + return + + max_depth = project["file_structure"]["max_depth"] + gitignore_raw = project["file_structure"]["gitignore"] + + folder = projects.get_project_folder(project_name) + if runtime.is_development(): + folder = files.normalize_a0_path(folder) + + file_structure = projects.get_file_structure(project_name) + else: + set = settings.get_settings_for_prompt() + enabled = bool(set["workdir_show"]) + + if not enabled: + return + + max_depth = set["workdir_max_depth"] + max_files = set["workdir_max_files"] + max_folders = set["workdir_max_folders"] + max_lines = set["workdir_max_lines"] + gitignore_raw = set["workdir_gitignore"] + + folder = set["workdir_path"] + scan_path = files.get_abs_path_development(folder) + + files.create_dir(scan_path) + + file_structure = str( + file_tree.file_tree( + scan_path, + max_depth=max_depth, + max_files=max_files, + max_folders=max_folders, + max_lines=max_lines, + ignore=gitignore_raw, + output_mode=file_tree.OUTPUT_MODE_STRING, + ) + ) + + gitignore = cleanup_gitignore(gitignore_raw) + + file_structure_prompt = self.agent.read_prompt( + "agent.extras.workdir_structure.md", + max_depth=max_depth, + gitignore=gitignore, + folder=folder, + file_structure=file_structure, + ) + + loop_data.extras_temporary["project_file_structure"] = file_structure_prompt + + +def cleanup_gitignore(gitignore_raw: str) -> str: + """Process gitignore: split lines, strip, remove comments, remove empty lines.""" + gitignore_lines = [] + for line in gitignore_raw.split('\n'): + # Strip whitespace + line = line.strip() + # Remove inline comments (everything after #) + if '#' in line: + line = line.split('#')[0].strip() + # Keep only non-empty lines + if line: + gitignore_lines.append(line) + + return '\n'.join(gitignore_lines) if gitignore_lines else "nothing ignored" diff --git a/extensions/python/message_loop_prompts_before/AGENTS.md b/extensions/python/message_loop_prompts_before/AGENTS.md index 4938f736a9440cd9f47f5bb7fccd173b3282f40c..3483592c7098fe1f49b042b147bce438664f0744 100644 --- a/extensions/python/message_loop_prompts_before/AGENTS.md +++ b/extensions/python/message_loop_prompts_before/AGENTS.md @@ -1,3 +1,26 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:84ec20842d08ebea0e8f684a9e2ca88b1ae9a41f57e2a9467c6c68442595a91f -size 578 +# Message Loop Prompts Before Extensions DOX + +## Purpose + +- Own preprocessing before message-loop prompt construction. + +## Ownership + +- Ordered Python files own history organization waits and related prompt-preparation gates. + +## Local Contracts + +- Preserve history consistency before prompts are assembled. +- Avoid blocking indefinitely on background organization tasks. + +## Work Guidance + +- Keep waiting behavior observable and bounded. + +## Verification + +- Smoke-test prompt construction after chats with pending history organization. + +## Child DOX Index + +No child DOX files. diff --git a/extensions/python/message_loop_prompts_before/_90_organize_history_wait.py b/extensions/python/message_loop_prompts_before/_90_organize_history_wait.py index f33cada195b90450e78c14170b000623897fd4e2..08966d2b1db730da02c058b0220afa07c2824c64 100644 --- a/extensions/python/message_loop_prompts_before/_90_organize_history_wait.py +++ b/extensions/python/message_loop_prompts_before/_90_organize_history_wait.py @@ -1,3 +1,67 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:11ccd593776872e26c8501414b3847ec7f26c90661d81f827ad6258a708b1da4 -size 2453 +from helpers.extension import Extension +from agent import LoopData +from extensions.python.message_loop_end._10_organize_history import ( + DATA_NAME_TASK, + compress_history, +) +from helpers.defer import DeferredTask, THREAD_BACKGROUND + +MAX_SYNC_COMPRESSION_PASSES = 64 + + +class OrganizeHistoryWait(Extension): + async def execute(self, loop_data: LoopData = LoopData(), **kwargs): + if not self.agent: + return + + # sync action only required if the history is too large, otherwise leave it in background + passes = 0 + while self.agent.history.is_over_limit(): + passes += 1 + before_tokens = self.agent.history.get_tokens() + + # get task + task: DeferredTask | None = self.agent.get_data(DATA_NAME_TASK) + + # Check if the task is already done + if task: + if not task.is_ready(): + self.agent.context.log.set_progress("Compressing history...") + + # Wait for the task to complete + compressed = bool(await task.result()) + + # Clear the coroutine data after it's done + self.agent.set_data(DATA_NAME_TASK, None) + else: + # no task was running, start and wait + self.agent.context.log.set_progress("Compressing history...") + compressed = await compress_history(self.agent) + + after_tokens = self.agent.history.get_tokens() + if not compressed or after_tokens >= before_tokens: + self._log_compression_stalled(before_tokens, after_tokens) + break + + if passes >= MAX_SYNC_COMPRESSION_PASSES: + self._log_compression_stalled( + before_tokens, after_tokens, max_passes=True + ) + break + + def _log_compression_stalled( + self, before_tokens: int, after_tokens: int, max_passes: bool = False + ) -> None: + if not self.agent: + return + + detail = ( + f"History compression stopped after {MAX_SYNC_COMPRESSION_PASSES} passes" + if max_passes + else "History compression could not reduce the prompt history further" + ) + self.agent.context.log.log( + type="warning", + heading="History compression stalled", + content=f"{detail}. Tokens before: {before_tokens}; after: {after_tokens}.", + ) diff --git a/extensions/python/message_loop_result/AGENTS.md b/extensions/python/message_loop_result/AGENTS.md index ff01f244c7a3b9514d8be2bea516c1e0f7fbc653..c1395c5fb8e942b325bc81283fb85dcb44fbe234 100644 --- a/extensions/python/message_loop_result/AGENTS.md +++ b/extensions/python/message_loop_result/AGENTS.md @@ -1,3 +1,31 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:13ea49ad9c43187bd08b39a3ee2606e11927a6e70a4bd2125b4a1cce1d52516b -size 1458 +# Message Loop Result Extensions DOX + +## Purpose + +- Own normalization and policy handling after a model turn completes, before default assistant-history and tool-dispatch processing. + +## Ownership + +- Extensions receive mutable `result_data` with `llm_result` and may set `skip_default_processing` after fully handling the turn. +- `_20_empty_response.py` retries turns with neither response nor reasoning, counts them toward the unusable-response limit without adding a warning to model history, and uses `fw.msg_empty_response.md` for agent-prefixed UI warning text only. +- `_30_repeat_response.py` retries response content that exactly matches `loop_data.last_response`, regardless of reasoning, using `fw.msg_repeat.md` for history and `fw.msg_repeat_response.md` for the agent-prefixed UI warning text. + +## Local Contracts + +- Files run in deterministic filename order. +- A handler that sets `skip_default_processing` owns needed history and UI side effects for that turn. +- Handlers that should not add side effects after an earlier extension has handled the result must return when `skip_default_processing` is set. +- Do not use this point to mutate streamed partial content. + +## Work Guidance + +- Normalize a completed result before policy extensions compare or persist it. +- Keep loop-control policy independent from optional plugins. + +## Verification + +- Run message-loop and unusable-response regression tests. + +## Child DOX Index + +No child DOX files. diff --git a/extensions/python/message_loop_result/_20_empty_response.py b/extensions/python/message_loop_result/_20_empty_response.py index 958fe026f7a3e130d018117cc02ceb87190c7aa9..aa79bf8c8999a98510963f3c0ef3267c697d441e 100644 --- a/extensions/python/message_loop_result/_20_empty_response.py +++ b/extensions/python/message_loop_result/_20_empty_response.py @@ -1,3 +1,53 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:3ac3e39730ee55e33b28378a1a53a564812b20baa6857eac55164ca30acd78f5 -size 2074 +"""Retry completed model turns with neither response nor reasoning.""" + +from __future__ import annotations + +from typing import Any + +from helpers.errors import HandledException +from helpers.extension import Extension +from helpers.print_style import PrintStyle +from helpers.settings import get_settings + + +STATE_KEY = "_unusable_response_failures" + + +class EmptyResponse(Extension): + def execute(self, result_data: dict[str, Any] | None = None, **kwargs: Any) -> None: + if not self.agent or not isinstance(result_data, dict): + return + if result_data.get("skip_default_processing"): + return + + llm_result = result_data.get("llm_result") + response = getattr(llm_result, "response", "") + reasoning = getattr(llm_result, "reasoning", "") + if not isinstance(response, str) or not isinstance(reasoning, str): + return + if response.strip() or reasoning.strip(): + return + + warning = self.agent.read_prompt("fw.msg_empty_response.md") + PrintStyle(font_color="orange", padding=True).print(warning) + state = self.agent.loop_data.params_persistent + previous = state.get(STATE_KEY, {}) + previous_iteration = previous.get("iteration") if isinstance(previous, dict) else None + count = ( + previous.get("count", 0) + 1 + if previous_iteration == self.agent.loop_data.iteration - 1 + else 1 + ) + state[STATE_KEY] = {"iteration": self.agent.loop_data.iteration, "count": count} + limit = get_settings()["max_consecutive_unusable_responses"] + if count >= limit: + stop_message = self.agent.read_prompt( + "fw.msg_unusable_response_limit.md", limit=limit + ) + self.agent.context.log.log(type="warning", content=stop_message) + raise HandledException(stop_message) + self.agent.context.log.log( + type="warning", + content=f"{self.agent.agent_name}: {warning}", + ) + result_data["skip_default_processing"] = True diff --git a/extensions/python/message_loop_result/_30_repeat_response.py b/extensions/python/message_loop_result/_30_repeat_response.py index f06dc53316822264b446d678774988a1ff290e15..45457ff82440279275f7fd28bc066b9375317264 100644 --- a/extensions/python/message_loop_result/_30_repeat_response.py +++ b/extensions/python/message_loop_result/_30_repeat_response.py @@ -1,3 +1,38 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:829c800ff43a67bfa30e5f18dfd62598ac885b601533949fe0dc2ee599c04d65 -size 1514 +"""Retry completed model turns that repeat the prior response.""" + +from __future__ import annotations + +from typing import Any + +from helpers.extension import Extension +from helpers.print_style import PrintStyle + + +class RepeatResponse(Extension): + def execute(self, result_data: dict[str, Any] | None = None, **kwargs: Any) -> None: + if not self.agent or not isinstance(result_data, dict): + return + if result_data.get("skip_default_processing"): + return + + llm_result = result_data.get("llm_result") + response = getattr(llm_result, "response", "") + if not isinstance(response, str) or response != self.agent.loop_data.last_response: + return + + warning = self.agent.read_prompt("fw.msg_repeat.md") + log_item = self.agent.loop_data.params_temporary.get("log_item_generating") + assistant_message = self.agent.hist_add_ai_response( + response, + id=log_item.id if log_item else "", + llm_result=llm_result, + ) + self.agent._remember_llm_result_state(llm_result, assistant_message) + warning_message = self.agent.hist_add_warning(message=warning) + PrintStyle(font_color="orange", padding=True).print(warning) + self.agent.context.log.log( + type="warning", + content=f"{self.agent.agent_name}: {self.agent.read_prompt('fw.msg_repeat_response.md')}", + id=warning_message.id, + ) + result_data["skip_default_processing"] = True diff --git a/extensions/python/message_loop_start/AGENTS.md b/extensions/python/message_loop_start/AGENTS.md index 11ceba7b93ac1f06af3d01e1e0d0853c0ff0974a..a0c6c1790905b3f9ebaf8c71b139c54482dcc3dc 100644 --- a/extensions/python/message_loop_start/AGENTS.md +++ b/extensions/python/message_loop_start/AGENTS.md @@ -1,3 +1,26 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:293e468e821fbac3bc0fc1baaaf9c8678db6b281ffe69652cb351934b8caa0c4 -size 592 +# Message Loop Start Extensions DOX + +## Purpose + +- Own backend behavior that runs at the start of each message loop iteration. + +## Ownership + +- Ordered Python files own iteration counters and future loop-start state setup. + +## Local Contracts + +- Keep per-loop counters deterministic and scoped to the active context. +- Do not reset state owned by monologue-level hooks. + +## Work Guidance + +- Coordinate loop-start state changes with logging, streaming, and process-chain behavior. + +## Verification + +- Smoke-test multi-turn conversations after changes. + +## Child DOX Index + +No child DOX files. diff --git a/extensions/python/message_loop_start/_10_iteration_no.py b/extensions/python/message_loop_start/_10_iteration_no.py index 1b87750c976cb1d63c40f0e208fad6d7c375cf8d..81fc4af97566941adf45f447b3918f82beed89e7 100644 --- a/extensions/python/message_loop_start/_10_iteration_no.py +++ b/extensions/python/message_loop_start/_10_iteration_no.py @@ -1,3 +1,17 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:bd43316806328046d783e603e9620ce577395b218fa570fb3bc06fa3c9a3d005 -size 507 +from helpers.extension import Extension +from agent import Agent, LoopData + +DATA_NAME_ITER_NO = "iteration_no" + +class IterationNo(Extension): + async def execute(self, loop_data: LoopData = LoopData(), **kwargs): + if not self.agent: + return + + # total iteration number + no = self.agent.get_data(DATA_NAME_ITER_NO) or 0 + self.agent.set_data(DATA_NAME_ITER_NO, no + 1) + + +def get_iter_no(agent: Agent) -> int: + return agent.get_data(DATA_NAME_ITER_NO) or 0 \ No newline at end of file diff --git a/extensions/python/monologue_end/AGENTS.md b/extensions/python/monologue_end/AGENTS.md index 18939f31faba7aeef73ef94433f091c00167a02d..b842ba0711f63f1e238889091a236d69fc161b14 100644 --- a/extensions/python/monologue_end/AGENTS.md +++ b/extensions/python/monologue_end/AGENTS.md @@ -1,3 +1,26 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:3a6741c56324d4249821b4a744b4d1cafa912ee61c424b55b85c7b79540febed -size 576 +# Monologue End Extensions DOX + +## Purpose + +- Own backend behavior that runs when a monologue ends. + +## Ownership + +- Ordered Python files own waiting-for-input UI message behavior and future monologue-end cleanup. + +## Local Contracts + +- Preserve clear UI state when the agent stops for user input. +- Do not leave loading/processing indicators stale. + +## Work Guidance + +- Coordinate changes with WebUI loading state and message-loop persistence. + +## Verification + +- Smoke-test an agent response that returns to waiting-for-input state. + +## Child DOX Index + +No child DOX files. diff --git a/extensions/python/monologue_end/_90_waiting_for_input_msg.py b/extensions/python/monologue_end/_90_waiting_for_input_msg.py index e255ac24ddeaad5c46b304d11e0f32193d7a731d..09a025b1046b7b7dcc7601f514c06c8945bf8ac3 100644 --- a/extensions/python/monologue_end/_90_waiting_for_input_msg.py +++ b/extensions/python/monologue_end/_90_waiting_for_input_msg.py @@ -1,3 +1,12 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:6c783e2e86031a26d33ed50204fbeb28004e1813618f9eb5851e98f2a69712d5 -size 364 +from helpers.extension import Extension +from agent import LoopData + +class WaitingForInputMsg(Extension): + + async def execute(self, loop_data: LoopData = LoopData(), **kwargs): + if not self.agent: + return + + # show temp info message + if self.agent.number == 0: + self.agent.context.log.set_initial_progress() diff --git a/extensions/python/monologue_start/AGENTS.md b/extensions/python/monologue_start/AGENTS.md index 35412924c97fc7d9845964fe190f270be7cc0827..af75a96458c789a05f01b219ad18ee4bbee99378 100644 --- a/extensions/python/monologue_start/AGENTS.md +++ b/extensions/python/monologue_start/AGENTS.md @@ -1,3 +1,26 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:a127681e8f3a83a292232daea1386f094e8253a29192cc22fe28c42a9b0ba6d7 -size 598 +# Monologue Start Extensions DOX + +## Purpose + +- Own core backend behavior that runs when a monologue starts. + +## Ownership + +- Ordered Python files own core monologue-start setup. + +## Local Contracts + +- Keep start-of-monologue work bounded and non-blocking where appropriate. +- Plugin-specific behavior belongs in the owning plugin's `extensions/python/monologue_start/` directory. + +## Work Guidance + +- Coordinate lifecycle changes with the message loop and relevant plugin hooks. + +## Verification + +- Smoke-test the first monologue after a new user message. + +## Child DOX Index + +No child DOX files. diff --git a/extensions/python/process_chain_end/AGENTS.md b/extensions/python/process_chain_end/AGENTS.md index 17397ce0334b7eef37d97acaf4be03615900c14d..c703404175d82473ffbb02645c0f9e814e73c1f2 100644 --- a/extensions/python/process_chain_end/AGENTS.md +++ b/extensions/python/process_chain_end/AGENTS.md @@ -1,3 +1,26 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:189f8b5532b795535555b78d8f726d511bd8f0154e3aee65e924d854f03d4ae0 -size 626 +# Process Chain End Extensions DOX + +## Purpose + +- Own backend behavior after process-chain execution completes. + +## Ownership + +- Ordered Python files own queued-message processing and future process-chain completion behavior. + +## Local Contracts + +- Preserve queue ordering and avoid duplicate message processing. +- Keep queue side effects synchronized with chat persistence and WebUI state. + +## Work Guidance + +- Coordinate changes with message queue components and external integration plugins. + +## Verification + +- Smoke-test queued messages and final response handling after changes. + +## Child DOX Index + +No child DOX files. diff --git a/extensions/python/process_chain_end/_50_process_queue.py b/extensions/python/process_chain_end/_50_process_queue.py index 63ae00044a493356edde1d7f776d210689407b0c..ad2960be8e57b222106dc9314cb63386bd2e4faf 100644 --- a/extensions/python/process_chain_end/_50_process_queue.py +++ b/extensions/python/process_chain_end/_50_process_queue.py @@ -1,3 +1,39 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:af9c0676106e070f0db2b51b007a5ae330ffc73701768b1832192db8c6d91310 -size 1441 +import asyncio +from helpers.extension import Extension +from helpers import message_queue as mq +from agent import AgentContext, Agent, LoopData +from helpers.state_monitor_integration import mark_dirty_for_context + + +class ProcessQueue(Extension): + """Process queued messages after monologue ends.""" + + async def execute(self, loop_data: LoopData = LoopData(), **kwargs): + if not self.agent: + return + + # Only process for agent0 (main agent) + if self.agent.number != 0: + return + + context = self.agent.context + + # Check if there are queued messages + if mq.has_queue(context): + # Schedule delayed task to send next queued message + # This allows current monologue to fully complete first + asyncio.create_task(self._delayed_send(context)) + + async def _delayed_send(self, context: AgentContext): + """Wait for task to complete, then send next queued message.""" + + # Wait for current task to finish, but no more than 1 minute to prevent hanging tasks + total_wait = 0 + while context.is_running() and total_wait < 60: + await asyncio.sleep(0.1) + total_wait += 0.1 + + # Send next queued message if task is not running + if not context.is_running(): + if mq.send_next(context): + mark_dirty_for_context(context.id, reason="message_queue_auto_send") diff --git a/extensions/python/reasoning_stream/AGENTS.md b/extensions/python/reasoning_stream/AGENTS.md index ba1f3ab11767e2080665e471665f1df5019a70a5..6cd3e6b2e2bcb73e30d40b7796e624e453598535 100644 --- a/extensions/python/reasoning_stream/AGENTS.md +++ b/extensions/python/reasoning_stream/AGENTS.md @@ -1,3 +1,26 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:0b33ce0d7d9d5b546643cd89301af7f7995389be6d8f4bd6839001772d2da8be -size 576 +# Reasoning Stream Extensions DOX + +## Purpose + +- Own handling of full reasoning stream updates. + +## Ownership + +- Ordered Python files own logging reasoning content from stream state. + +## Local Contracts + +- Preserve masking and privacy rules for reasoning content. +- Keep stream logging compatible with chunk and end hooks. + +## Work Guidance + +- Coordinate reasoning stream changes with UI log rendering and hidden-content policy. + +## Verification + +- Smoke-test reasoning stream display/logging when the active model provides reasoning. + +## Child DOX Index + +No child DOX files. diff --git a/extensions/python/reasoning_stream/_10_log_from_stream.py b/extensions/python/reasoning_stream/_10_log_from_stream.py index bdd65838ff80a76c79225c5463ed4c23d379f264..8f228b9b1bfaeedb6a8d43ccb1146409392ea079 100644 --- a/extensions/python/reasoning_stream/_10_log_from_stream.py +++ b/extensions/python/reasoning_stream/_10_log_from_stream.py @@ -1,3 +1,34 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:2958f7d5cd63c8c18dc6205c2d86690fc59def12b8a4ce88fb593c167bb16fe7 -size 1269 +from helpers import persist_chat, tokens +from helpers.extension import Extension +from agent import LoopData +import asyncio +from helpers.log import LogItem +from helpers import log +import math +from extensions.python.before_main_llm_call._10_log_for_stream import build_heading, build_default_heading + +class LogFromStream(Extension): + + async def execute(self, loop_data: LoopData = LoopData(), text: str = "", **kwargs): + if not self.agent: + return + + # thought length indicator + length = f"({len(text)})" if text else "" + pipes = "|" * math.ceil(math.sqrt(len(text))/2) + heading = build_heading(self.agent, f"Reasoning... {pipes}") + step = f"Reasoning... {length}" + + # create log message and store it in loop data temporary params + if "log_item_generating" not in loop_data.params_temporary: + loop_data.params_temporary["log_item_generating"] = ( + self.agent.context.log.log( + type="agent", + heading=heading, + step=step + ) + ) + + # update log message + log_item = loop_data.params_temporary["log_item_generating"] + log_item.update(heading=heading, reasoning=text, step=step) diff --git a/extensions/python/reasoning_stream_chunk/AGENTS.md b/extensions/python/reasoning_stream_chunk/AGENTS.md index 32ba60aaf182064a49379ca1b0825946ed71e4d8..4e7461224b2bfc1594f7ac98c7696e31953bdd6d 100644 --- a/extensions/python/reasoning_stream_chunk/AGENTS.md +++ b/extensions/python/reasoning_stream_chunk/AGENTS.md @@ -1,3 +1,26 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:d3b83de9707a34473f0519ecc99c9863bc086b9f6d8d270b43085290ef0312ab -size 598 +# Reasoning Stream Chunk Extensions DOX + +## Purpose + +- Own handling of incremental reasoning stream chunks. + +## Ownership + +- Ordered Python files own chunk-level masking before reasoning is displayed or stored. + +## Local Contracts + +- Mask secrets and sensitive content before chunk data reaches logs or UI. +- Keep chunk mutation compatible with final stream-end masking. + +## Work Guidance + +- Keep chunk processing lightweight for streaming performance. + +## Verification + +- Smoke-test streamed reasoning with representative sensitive patterns after changes. + +## Child DOX Index + +No child DOX files. diff --git a/extensions/python/reasoning_stream_chunk/_10_mask_stream.py b/extensions/python/reasoning_stream_chunk/_10_mask_stream.py index 89ad5d6bba8d6d858e128fbf4f599de73943d587..bd3adc855d88cd8e86bf71aa2dde59c19fd1d5b4 100644 --- a/extensions/python/reasoning_stream_chunk/_10_mask_stream.py +++ b/extensions/python/reasoning_stream_chunk/_10_mask_stream.py @@ -1,3 +1,41 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:32c5e7b46fac58ba4ce38b204cb91dbf6552e0e1c3dfb9f513e59c374e40d359 -size 1540 +from helpers.extension import Extension +from helpers.secrets import get_secrets_manager + + +class MaskReasoningStreamChunk(Extension): + async def execute(self, **kwargs): + if not self.agent: + return + + # Get stream data and agent from kwargs + stream_data = kwargs.get("stream_data") + agent = kwargs.get("agent") + if not agent or not stream_data: + return + + try: + secrets_mgr = get_secrets_manager(self.agent.context) + + # Initialize filter if not exists + filter_key = "_reason_stream_filter" + filter_instance = agent.get_data(filter_key) + if not filter_instance: + filter_instance = secrets_mgr.create_streaming_filter() + agent.set_data(filter_key, filter_instance) + + # Process the chunk through the streaming filter + processed_chunk = filter_instance.process_chunk(stream_data["chunk"]) + + # Update the stream data with processed chunk + stream_data["chunk"] = processed_chunk + + # Also mask the full text for consistency + stream_data["full"] = secrets_mgr.mask_values(stream_data["full"]) + + # Print the processed chunk (this is where printing should happen) + if processed_chunk: + from helpers.print_style import PrintStyle + PrintStyle().stream(processed_chunk) + except Exception as e: + # If masking fails, proceed without masking + pass diff --git a/extensions/python/reasoning_stream_end/AGENTS.md b/extensions/python/reasoning_stream_end/AGENTS.md index 53ff5bbc1171dd68411eaf6cbea2b49d966f71f5..d702b22ad8153de579860079bedabf76fbe1a473 100644 --- a/extensions/python/reasoning_stream_end/AGENTS.md +++ b/extensions/python/reasoning_stream_end/AGENTS.md @@ -1,3 +1,26 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:88d95271df5554a22510896b9cc7ec04631b8e988468b82f9d5e20aa55e369c4 -size 568 +# Reasoning Stream End Extensions DOX + +## Purpose + +- Own finalization of reasoning stream content. + +## Ownership + +- Ordered Python files own final reasoning masking and end-of-stream cleanup. + +## Local Contracts + +- Preserve final masking even if earlier chunk masking missed content. +- Keep end-state consistent with reasoning stream log entries. + +## Work Guidance + +- Coordinate final masking changes with chunk masking and UI rendering. + +## Verification + +- Smoke-test reasoning stream completion with sensitive-content cases. + +## Child DOX Index + +No child DOX files. diff --git a/extensions/python/reasoning_stream_end/_10_mask_end.py b/extensions/python/reasoning_stream_end/_10_mask_end.py index 3a6025c98fe196a986ff64168ef419a17f3faa56..89787c4938d8b32cd4b32480a0f82dba4be0ce68 100644 --- a/extensions/python/reasoning_stream_end/_10_mask_end.py +++ b/extensions/python/reasoning_stream_end/_10_mask_end.py @@ -1,3 +1,27 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:55eae68432367201f292a4bdd13527519d763f45b2247a20cb09901619d3c872 -size 899 +from helpers.extension import Extension + + +class MaskReasoningStreamEnd(Extension): + async def execute(self, **kwargs): + # Get agent and finalize the streaming filter + agent = kwargs.get("agent") + if not agent: + return + + try: + # Finalize the reasoning stream filter if it exists + filter_key = "_reason_stream_filter" + filter_instance = agent.get_data(filter_key) + if filter_instance: + tail = filter_instance.finalize() + + # Print any remaining masked content + if tail: + from helpers.print_style import PrintStyle + PrintStyle().stream(tail) + + # Clean up the filter + agent.set_data(filter_key, None) + except Exception as e: + # If masking fails, proceed without masking + pass diff --git a/extensions/python/response_stream/AGENTS.md b/extensions/python/response_stream/AGENTS.md index 4c940699cc0fcd614c30127f1a6bb1beeb21b0c9..5da570f6bfe95bae1771bdd4a0fb6df642da46ea 100644 --- a/extensions/python/response_stream/AGENTS.md +++ b/extensions/python/response_stream/AGENTS.md @@ -1,3 +1,29 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:ef5a18977aa5381108e71b0c6efff2c7e7429f9abe709ccc4a85264cdd30485d -size 781 +# Response Stream Extensions DOX + +## Purpose + +- Own handling of full assistant response stream updates. + +## Ownership + +- Ordered Python files own response logging, include-alias replacement, and live response updates. + +## Local Contracts + +- Keep streaming output synchronized with UI log items. +- Treat parsed stream snapshots as partial data; nested tool fields may be `None` + until their values arrive. +- Preserve include-alias replacement semantics where prompts/tools rely on them. +- Do not expose unmasked secrets in live responses. + +## Work Guidance + +- Coordinate stream changes with chunk/end hooks and message rendering. + +## Verification + +- Smoke-test streamed responses, live updates, and include alias replacement after changes. + +## Child DOX Index + +No child DOX files. diff --git a/extensions/python/response_stream/_10_log_from_stream.py b/extensions/python/response_stream/_10_log_from_stream.py index 40507a95bc95014848c71208690f7832a17d786b..91621dbe9bbf3b38ce44c9f85422cf6a62906e68 100644 --- a/extensions/python/response_stream/_10_log_from_stream.py +++ b/extensions/python/response_stream/_10_log_from_stream.py @@ -1,3 +1,74 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:373f6093952c70a902b9d4c82942c06d5d59da6fb8f1c603a15932c048696565 -size 2988 +from helpers import persist_chat, tokens +from helpers.extension import Extension +from agent import LoopData +import asyncio +from helpers.log import LogItem +from helpers import log +import math +from extensions.python.before_main_llm_call._10_log_for_stream import build_heading, build_default_heading + + +class LogFromStream(Extension): + + async def execute( + self, + loop_data: LoopData = LoopData(), + text: str = "", + parsed: dict = {}, + **kwargs, + ): + if not self.agent: + return + + heading = build_default_heading(self.agent) + if "headline" in parsed: + heading = build_heading(self.agent, parsed['headline']) + elif "tool_name" in parsed: + heading = build_heading(self.agent, f"Using {parsed['tool_name']}") # if the llm skipped headline + elif "thoughts" in parsed: + # thought length indicator + length = "|" * math.ceil(math.sqrt(len(text))/2) + heading = build_heading(self.agent, f"Thinking... {length}") + else: + heading = build_heading(self.agent, "Receiving...") + + # create log message and store it in loop data temporary params + if "log_item_generating" not in loop_data.params_temporary: + loop_data.params_temporary["log_item_generating"] = ( + self.agent.context.log.log( + type="agent", + heading=heading, + ) + ) + + # update log message + log_item = loop_data.params_temporary["log_item_generating"] + + # keep reasoning from previous logs in kvps + kvps = {} + if log_item.kvps is not None and "reasoning" in log_item.kvps: + kvps["reasoning"] = log_item.kvps["reasoning"] + + # step description for UI - using tool XY, writing Python code, etc. + if parsed is not None and "tool_name" in parsed and parsed["tool_name"]: + kvps["step"] = f"Using {parsed['tool_name']}..." # using tool XY + if parsed["tool_name"]=="code_execution_tool": + tool_args = parsed.get("tool_args") + if isinstance(tool_args, dict) and "runtime" in tool_args: + length = "" + code = tool_args.get("code") + if isinstance(code, str): + length = f"({len(code)})" + kvps["step"] = f"Writing code... {length}" + if tool_args["runtime"] == "python": + kvps["step"] = f"Writing Python code... {length}" + elif tool_args["runtime"] == "nodejs": + kvps["step"] = f"Writing Node.js code... {length}" + elif tool_args["runtime"] == "terminal": + kvps["step"] = f"Writing terminal command... {length}" + kvps.update(parsed) + + + + # update the log item + log_item.update(heading=heading, content=text, kvps=kvps) diff --git a/extensions/python/response_stream/_15_replace_include_alias.py b/extensions/python/response_stream/_15_replace_include_alias.py index abf3fc3f75c25d2c1c139bd2f2dd43f3cf4e15b5..0994b75f6abf1d3bd4bb9b2ab583d60af93c0db5 100644 --- a/extensions/python/response_stream/_15_replace_include_alias.py +++ b/extensions/python/response_stream/_15_replace_include_alias.py @@ -1,3 +1,31 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:33182c00a343a86bcd80aaa0b561098e36efcfc1cf93ab7b38faab9a1c409b81 -size 1116 +from typing import Any +from helpers.extension import Extension +from helpers.strings import replace_file_includes + + +class ReplaceIncludeAlias(Extension): + async def execute( + self, + loop_data=None, + text: str = "", + parsed: dict[str, Any] | None = None, + **kwargs + ): + if not parsed or not isinstance(parsed, dict): + return + + def replace_placeholders(value: Any) -> Any: + if isinstance(value, str): + new_val = value + new_val = replace_file_includes(new_val, r"§§include\(([^)]+)\)") + return new_val + if isinstance(value, dict): + return {k: replace_placeholders(v) for k, v in value.items()} + if isinstance(value, list): + return [replace_placeholders(v) for v in value] + if isinstance(value, tuple): + return tuple(replace_placeholders(v) for v in value) + return value + + if "tool_args" in parsed and "tool_name" in parsed: + parsed["tool_args"] = replace_placeholders(parsed["tool_args"]) diff --git a/extensions/python/response_stream/_20_live_response.py b/extensions/python/response_stream/_20_live_response.py index 9d0e6c56d1eed705ded732abec440b3751c825b3..f206bd98f7dd70011a6b55ac9f8ff3e9cd7dfdc6 100644 --- a/extensions/python/response_stream/_20_live_response.py +++ b/extensions/python/response_stream/_20_live_response.py @@ -1,3 +1,51 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:6fd60016910f1ff691d9e70047817edb560aa0b3b5f2b1180045bb5867f4489d -size 1829 +from helpers import persist_chat, tokens +from helpers import extract_tools +from helpers.extension import Extension +from agent import LoopData +import asyncio +from helpers.log import LogItem +from helpers import log + + +class LiveResponse(Extension): + + async def execute( + self, + loop_data: LoopData = LoopData(), + text: str = "", + parsed: dict = {}, + **kwargs, + ): + if not self.agent: + return + + try: + tool_name, tool_args = extract_tools.normalize_tool_request(parsed) + message = tool_args.get("text") + if not isinstance(message, str) or not message.strip(): + message = tool_args.get("message") + if ( + tool_name != "response" + or not isinstance(message, str) + or not message.strip() + ): + return # not a response + + # create log message and store it in loop data temporary params + if "log_item_response" not in loop_data.params_temporary: + # Share id with the agent log item so branching covers the response bubble + gen_item = loop_data.params_temporary.get("log_item_generating") + shared_id = gen_item.id if gen_item and gen_item.id else "" + loop_data.params_temporary["log_item_response"] = ( + self.agent.context.log.log( + type="response", + heading=f"icon://chat {self.agent.agent_name}: Responding", + id=shared_id, + ) + ) + + # update log message + log_item = loop_data.params_temporary["log_item_response"] + log_item.update(content=message) + except Exception as e: + pass diff --git a/extensions/python/response_stream_chunk/AGENTS.md b/extensions/python/response_stream_chunk/AGENTS.md index f3e316904208dc18979b49da1247f3def3500a1a..9ceff97a6241a45216545674731f9eaaf0088223 100644 --- a/extensions/python/response_stream_chunk/AGENTS.md +++ b/extensions/python/response_stream_chunk/AGENTS.md @@ -1,3 +1,26 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:699283534dc6da10e7b6cec11fddddd4e2e50fd02bdf6e3887555d6ed1ed9281 -size 553 +# Response Stream Chunk Extensions DOX + +## Purpose + +- Own handling of incremental assistant response chunks. + +## Ownership + +- Ordered Python files own chunk-level response masking. + +## Local Contracts + +- Mask secrets before response chunks reach UI or persisted logs. +- Keep chunk processing compatible with final response stream masking. + +## Work Guidance + +- Keep per-chunk work lightweight for streaming responsiveness. + +## Verification + +- Smoke-test streaming responses with sensitive patterns after changes. + +## Child DOX Index + +No child DOX files. diff --git a/extensions/python/response_stream_chunk/_10_mask_stream.py b/extensions/python/response_stream_chunk/_10_mask_stream.py index 55f0e50e16fba669057e31862d0ac3938b54980d..4690df662b0d25aa1fe36a1ba82883476566b814 100644 --- a/extensions/python/response_stream_chunk/_10_mask_stream.py +++ b/extensions/python/response_stream_chunk/_10_mask_stream.py @@ -1,3 +1,43 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:bd11cfb50370d874523f82f586b92a03b192eb995247ff649839eeeb62057789 -size 1560 +from helpers.extension import Extension +from agent import Agent, LoopData +from helpers.secrets import get_secrets_manager + + +class MaskResponseStreamChunk(Extension): + + async def execute(self, **kwargs): + if not self.agent: + return + + # Get stream data and agent from kwargs + stream_data = kwargs.get("stream_data") + agent = kwargs.get("agent") + if not agent or not stream_data: + return + + try: + secrets_mgr = get_secrets_manager(self.agent.context) + + # Initialize filter if not exists + filter_key = "_resp_stream_filter" + filter_instance = agent.get_data(filter_key) + if not filter_instance: + filter_instance = secrets_mgr.create_streaming_filter() + agent.set_data(filter_key, filter_instance) + + # Process the chunk through the streaming filter + processed_chunk = filter_instance.process_chunk(stream_data["chunk"]) + + # Update the stream data with processed chunk + stream_data["chunk"] = processed_chunk + + # Also mask the full text for consistency + stream_data["full"] = secrets_mgr.mask_values(stream_data["full"]) + + # Print the processed chunk (this is where printing should happen) + if processed_chunk: + from helpers.print_style import PrintStyle + PrintStyle().stream(processed_chunk) + except Exception as e: + # If masking fails, proceed without masking + pass diff --git a/extensions/python/response_stream_end/AGENTS.md b/extensions/python/response_stream_end/AGENTS.md index 9f1de8dc79c14d2ccda1aae47db951d3b2d9916f..b11052105a7bd572f6209cdd7b2dc074c37b22f2 100644 --- a/extensions/python/response_stream_end/AGENTS.md +++ b/extensions/python/response_stream_end/AGENTS.md @@ -1,3 +1,28 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:11e9948bcd2a5898feee819a0241b3c52303c577507c5b26133e4a7976019a07 -size 740 +# Response Stream End Extensions DOX + +## Purpose + +- Own finalization of assistant response stream content. + +## Ownership + +- Ordered Python files own final masking and stream-end log updates. + +## Local Contracts + +- Preserve final masking before response content is considered complete. +- Keep log state consistent with streamed chunks and final response text. +- Preserve explicit planning fields; when `thoughts` are absent, expose native + reasoning summaries as thoughts after both streams finish. + +## Work Guidance + +- Coordinate finalization changes with live response and message rendering behavior. + +## Verification + +- Smoke-test response completion and persisted message display after changes. + +## Child DOX Index + +No child DOX files. diff --git a/extensions/python/response_stream_end/_10_mask_end.py b/extensions/python/response_stream_end/_10_mask_end.py index d311d14c49f0d48ac0269bbdf1830c8d2b488fe8..1d277331999fbdc1fedd6ff11bc196779dd6a730 100644 --- a/extensions/python/response_stream_end/_10_mask_end.py +++ b/extensions/python/response_stream_end/_10_mask_end.py @@ -1,3 +1,28 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:5f7fa3c82c5d468fd680127251a50d3cc189153c55e1c10d038635619eab035d -size 938 +from helpers.extension import Extension +from helpers.secrets import SecretsManager + + +class MaskResponseStreamEnd(Extension): + async def execute(self, **kwargs): + # Get agent and finalize the streaming filter + agent = kwargs.get("agent") + if not agent: + return + + try: + # Finalize the response stream filter if it exists + filter_key = "_resp_stream_filter" + filter_instance = agent.get_data(filter_key) + if filter_instance: + tail = filter_instance.finalize() + + # Print any remaining masked content + if tail: + from helpers.print_style import PrintStyle + PrintStyle().stream(tail) + + # Clean up the filter + agent.set_data(filter_key, None) + except Exception as e: + # If masking fails, proceed without masking + pass diff --git a/extensions/python/response_stream_end/_15_log_from_stream_end.py b/extensions/python/response_stream_end/_15_log_from_stream_end.py index edc73333c5841b4558d24791653bae12cded7c48..a23826e9534b38c8cc63c45a8cb0b0d0c34754fe 100644 --- a/extensions/python/response_stream_end/_15_log_from_stream_end.py +++ b/extensions/python/response_stream_end/_15_log_from_stream_end.py @@ -1,3 +1,36 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:268417889e71053f838bce32ea1d6d4384b1a092fea330bc90a33c0a4d238716 -size 1095 +from helpers import persist_chat, tokens +from helpers.extension import Extension +from agent import LoopData +import asyncio +from helpers.log import LogItem +from helpers import log +import math +from extensions.python.before_main_llm_call._10_log_for_stream import build_heading, build_default_heading + + +class LogFromStream(Extension): + + async def execute( + self, + loop_data: LoopData = LoopData(), + text: str = "", + parsed: dict = {}, + **kwargs, + ): + + # get log item from loop data temporary params + log_item = loop_data.params_temporary["log_item_generating"] + if log_item is None: + return + + # remove step parameter when done + if log_item.kvps is not None and "step" in log_item.kvps: + del log_item.kvps["step"] + + if log_item.kvps is not None and "thoughts" not in log_item.kvps: + reasoning = log_item.kvps.get("reasoning") + if reasoning: + log_item.kvps["thoughts"] = [reasoning] + + # update the log item + log_item.update(kvps=log_item.kvps) diff --git a/extensions/python/startup_migration/AGENTS.md b/extensions/python/startup_migration/AGENTS.md index 7c1c22f30f6442e4f9521cd03fac862ef12037c4..d3ffc39a09938cf82523181773e1156e7e745e1f 100644 --- a/extensions/python/startup_migration/AGENTS.md +++ b/extensions/python/startup_migration/AGENTS.md @@ -1,3 +1,29 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:887b991895140d9b98d248cf6db5d4e2058597c1eb0a10eb2e99b73677821f36 -size 1104 +# Startup Migration Extensions DOX + +## Purpose + +- Own backend startup migrations. + +## Ownership + +- Ordered Python files in this folder own idempotent migration steps that run during startup. + +## Local Contracts + +- Migrations must be safe to run repeatedly. +- Preserve user data and create backups or reversible paths when changing durable state. +- Keep long-running work bounded and observable. +- `_10_self_update_manager.py` may replace `/exe/self_update_manager.py` from the repository copy when the installed runtime updater is stale; it must validate required safety markers and keep a backup before replacement. +- After synchronizing a stale self-update manager, `_10_self_update_manager.py` starts that manager's best-effort Codex CLI refresh in the background so the update that introduces the hook does not need a second restart. + +## Work Guidance + +- Add migrations only for durable state changes that cannot be handled lazily elsewhere. + +## Verification + +- Smoke-test startup on a clean checkout and on representative existing user state when practical. + +## Child DOX Index + +No child DOX files. diff --git a/extensions/python/startup_migration/_10_self_update_manager.py b/extensions/python/startup_migration/_10_self_update_manager.py index ee570ea27bd26825c8f8afbbd3d95c6cef05a4a7..fff24cc1cb6f44eba811cb4c96363138c6c080bd 100644 --- a/extensions/python/startup_migration/_10_self_update_manager.py +++ b/extensions/python/startup_migration/_10_self_update_manager.py @@ -1,3 +1,146 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:5c6d2e6497d4fc31ac0479b88ceff22930c6b592428141c772bebaf2dadff632 -size 4755 +from __future__ import annotations + +import os +import shutil +import stat +import subprocess +import sys +from pathlib import Path +from typing import Any + +from helpers.extension import Extension +from helpers.print_style import PrintStyle + + +SELF_UPDATE_MANAGER_PATH = Path( + os.environ.get("A0_SELF_UPDATE_MANAGER_PATH", "/exe/self_update_manager.py") +) +SELF_UPDATE_MANAGER_SOURCE_PATH = Path( + os.environ.get( + "A0_SELF_UPDATE_MANAGER_SOURCE_PATH", + "/a0/docker/run/fs/exe/self_update_manager.py", + ) +) +BACKUP_SUFFIX = ".startup-migration-backup" +REQUIRED_RUNTIME_MARKERS = ( + "def should_include_usr_backup_entry(", + "Skipping non-regular usr backup entry", + "def clean_transient_desktop_agent_state(", + "clean_transient_desktop_agent_state(REPO_DIR, logger)", + "def refresh_codex_cli(", + "refresh_codex_cli(logger)", +) + + +class SelfUpdateManagerRuntimeSync(Extension): + def execute(self, **kwargs): + result = ensure_self_update_manager_runtime_current() + if result.get("updated"): + PrintStyle.info("Self-update manager runtime synchronized:", result["target"]) + warning = start_codex_cli_refresh(result["target"]) + if warning: + PrintStyle.warning("Codex CLI refresh could not be started:", warning) + elif result.get("warning"): + PrintStyle.warning("Self-update manager runtime sync skipped:", result["warning"]) + + +def ensure_self_update_manager_runtime_current( + *, + target_path: Path | str | None = None, + source_path: Path | str | None = None, +) -> dict[str, Any]: + target = Path(target_path) if target_path is not None else SELF_UPDATE_MANAGER_PATH + source = Path(source_path) if source_path is not None else SELF_UPDATE_MANAGER_SOURCE_PATH + + target_text, target_warning = _read_regular_text(target, role="runtime self-update manager") + if target_text is None: + return {"ok": True, "updated": False, "reason": target_warning} + + source_text, source_warning = _read_regular_text(source, role="source self-update manager") + if source_text is None: + return {"ok": False, "updated": False, "warning": source_warning} + + missing_source_markers = _missing_required_markers(source_text) + if missing_source_markers: + return { + "ok": False, + "updated": False, + "warning": ( + "source self-update manager is missing required safety markers: " + + ", ".join(missing_source_markers) + ), + } + + if not _missing_required_markers(target_text): + return {"ok": True, "updated": False, "reason": "already-current"} + + try: + backup = _replace_runtime_manager(target, source_text) + except OSError as exc: + return { + "ok": False, + "updated": False, + "warning": f"could not update {target}: {exc}", + } + + return { + "ok": True, + "updated": True, + "target": str(target), + "backup": str(backup), + } + + +def start_codex_cli_refresh(manager_path: Path | str) -> str: + try: + subprocess.Popen( + [sys.executable, str(manager_path), "refresh-codex"], + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + start_new_session=True, + ) + except OSError as exc: + return str(exc) + return "" + + +def _missing_required_markers(text: str) -> list[str]: + return [marker for marker in REQUIRED_RUNTIME_MARKERS if marker not in text] + + +def _read_regular_text(path: Path, *, role: str) -> tuple[str | None, str]: + try: + path_stat = path.lstat() + except FileNotFoundError: + return None, f"{role} not found: {path}" + except OSError as exc: + return None, f"{role} could not be inspected: {path}: {exc}" + + if not stat.S_ISREG(path_stat.st_mode): + return None, f"{role} is not a regular file: {path}" + + try: + return path.read_text(encoding="utf-8"), "" + except OSError as exc: + return None, f"{role} could not be read: {path}: {exc}" + + +def _replace_runtime_manager(target: Path, source_text: str) -> Path: + target_stat = target.stat() + backup = _ensure_backup(target) + temp_path = target.with_name(f".{target.name}.{os.getpid()}.tmp") + try: + temp_path.write_text(source_text, encoding="utf-8") + os.chmod(temp_path, stat.S_IMODE(target_stat.st_mode)) + os.replace(temp_path, target) + finally: + temp_path.unlink(missing_ok=True) + return backup + + +def _ensure_backup(target: Path) -> Path: + backup = target.with_name(f"{target.name}{BACKUP_SUFFIX}") + if not backup.exists(): + shutil.copy2(target, backup) + return backup diff --git a/extensions/python/system_prompt/AGENTS.md b/extensions/python/system_prompt/AGENTS.md index 667942ff265f3ec6f1be800c8d7300fd9648163a..2edb213eb2fed5719070d56be6cfa76e452416c1 100644 --- a/extensions/python/system_prompt/AGENTS.md +++ b/extensions/python/system_prompt/AGENTS.md @@ -1,3 +1,32 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:96726b0ba326dd8a294277b9e99b7c921e024369038049f9a39cc426bd610a04 -size 1115 +# System Prompt Extensions DOX + +## Purpose + +- Own construction of core system prompt sections. + +## Ownership + +- Ordered Python files own main, tools, MCP, secrets, skills, and project prompt sections. +- Active project instruction bodies and active-project AGENTS.md path-chain guidance are moved into prompt protocol; the system prompt keeps project metadata and stable project rules. + +## Local Contracts + +- Preserve ordering where sections depend on earlier context. +- Keep secret-related prompt sections masked and scoped. +- Prompt additions must be bounded and compatible with tool-call contracts. +- Discover local tool prompts through `helpers.subagents.get_paths` and apply + `helpers.tool_policy` before including their text. +- Omit the discoverable-skills catalog when profile policy blocks + `skills_tool`; loaded skill history remains independent. + +## Work Guidance + +- Coordinate broad system prompt changes with profiles, skills, tools, plugins, and prompt tests. + +## Verification + +- Inspect rendered system prompts or run prompt-construction tests after changes. + +## Child DOX Index + +No child DOX files. diff --git a/extensions/python/system_prompt/_10_main_prompt.py b/extensions/python/system_prompt/_10_main_prompt.py index eea4d1094994169707bf0ba01901077d0a5f3320..88193540cc55bb69e515a0460d621ffd66c42c23 100644 --- a/extensions/python/system_prompt/_10_main_prompt.py +++ b/extensions/python/system_prompt/_10_main_prompt.py @@ -1,3 +1,23 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:dcbca9cae15e9ab7489611a08026ab8c20a861a17d0b867d359c3025fea42998 -size 533 +from typing import Any + +from helpers.extension import Extension, extensible +from agent import Agent, LoopData + + +class MainPrompt(Extension): + + async def execute( + self, + system_prompt: list[str] = [], + loop_data: LoopData = LoopData(), + **kwargs: Any, + ): + if not self.agent: + return + prompt = await build_prompt(self.agent) + system_prompt.append(prompt) + + +@extensible +async def build_prompt(agent: Agent) -> str: + return agent.read_prompt("agent.system.main.md") diff --git a/extensions/python/system_prompt/_11_tools_prompt.py b/extensions/python/system_prompt/_11_tools_prompt.py index 570cbff98d3373281299e7f4a1b35dfc06942a2f..bb6de55e6950a96c4513c5aab8e6d933c3c2dd08 100644 --- a/extensions/python/system_prompt/_11_tools_prompt.py +++ b/extensions/python/system_prompt/_11_tools_prompt.py @@ -1,3 +1,63 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:31447082ddac66afa7acfbd5789d801e7af13228ebebfad7411b1d6cdfb469f3 -size 1961 +import os +from typing import Any + +from helpers.extension import Extension, extensible +from helpers import files, subagents, tool_policy +from helpers.print_style import PrintStyle +from agent import Agent, LoopData + + +TOOL_KWARGS_KEY = "_tool_prompt_kwargs" + + +class ToolsPrompt(Extension): + + async def execute( + self, + system_prompt: list[str] = [], + loop_data: LoopData = LoopData(), + **kwargs: Any, + ): + if not self.agent: + return + prompt = await build_prompt(self.agent) + system_prompt.append(prompt) + + +@extensible +async def build_prompt(agent: Agent) -> str: + # collect tool files from all prompt directories + prompt_dirs = subagents.get_paths(agent, "prompts") + tool_files = files.get_unique_filenames_in_dirs( + prompt_dirs, "agent.system.tool.*.md" + ) + + # per-file kwargs registered by plugin config extensions (e.g. _09_text_editor_config) + all_tool_kwargs: dict[str, dict[str, Any]] = agent.get_data(TOOL_KWARGS_KEY) or {} + + tools: list[str] = [] + for tool_file in tool_files: + try: + basename = os.path.basename(tool_file) + extra = all_tool_kwargs.get(basename, {}) + tool = agent.read_prompt(basename, **extra) + tool = tool_policy.filter_tool_prompt(agent, basename, tool) + if tool: + tools.append(tool) + except Exception as e: + PrintStyle().error(f"Error loading tool '{tool_file}': {e}") + + tools_str = "\n\n".join(tools) + prompt = agent.read_prompt("agent.system.tools.md", tools=tools_str) + + # vision support + from plugins._model_config.helpers.model_config import ( + get_chat_model_config, + get_vision_model_config, + ) + + chat_cfg = get_chat_model_config(agent) + if get_vision_model_config(agent) or chat_cfg.get("vision", False): + prompt += "\n\n" + agent.read_prompt("agent.system.tools_vision.md") + + return prompt diff --git a/extensions/python/system_prompt/_12_mcp_prompt.py b/extensions/python/system_prompt/_12_mcp_prompt.py index 86aafedce7972cbb48bc15002b9699b6276efeba..c18c11fceaee05b29848a1d3c72fc8fb1642f0ad 100644 --- a/extensions/python/system_prompt/_12_mcp_prompt.py +++ b/extensions/python/system_prompt/_12_mcp_prompt.py @@ -1,3 +1,33 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:9e0000546d286090f1db3d0b0930a0db976164762b353cce2cd64e2c458b5fe4 -size 871 +from typing import Any + +from helpers.extension import Extension, extensible +from helpers.mcp_handler import MCPConfig +from agent import Agent, LoopData + + +class MCPToolsPrompt(Extension): + + async def execute( + self, + system_prompt: list[str] = [], + loop_data: LoopData = LoopData(), + **kwargs: Any, + ): + if not self.agent: + return + prompt = await build_prompt(self.agent) + if prompt: + system_prompt.append(prompt) + + +@extensible +async def build_prompt(agent: Agent) -> str: + mcp_config = MCPConfig.get_for_agent(agent) + if not mcp_config.servers: + return "" + + pre_progress = agent.context.log.progress + agent.context.log.set_progress("Collecting MCP tools") + tools = mcp_config.get_tools_prompt(agent=agent) + agent.context.log.set_progress(pre_progress) + return tools diff --git a/extensions/python/system_prompt/_13_secrets_prompt.py b/extensions/python/system_prompt/_13_secrets_prompt.py index 5f50361e71cb2f30e007b05b5b99aeab1381fde6..fdd4cfdd8f95781e5f94497fef6fabd0930c6801 100644 --- a/extensions/python/system_prompt/_13_secrets_prompt.py +++ b/extensions/python/system_prompt/_13_secrets_prompt.py @@ -1,3 +1,35 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:e74ac4ba67be562a78a3eb9b52c7ed667854e0d5af6daf4e75336d439009f62f -size 967 +from typing import Any + +from helpers.extension import Extension, extensible +from agent import Agent, LoopData + + +class SecretsPrompt(Extension): + + async def execute( + self, + system_prompt: list[str] = [], + loop_data: LoopData = LoopData(), + **kwargs: Any, + ): + if not self.agent: + return + prompt = await build_prompt(self.agent) + if prompt: + system_prompt.append(prompt) + + +@extensible +async def build_prompt(agent: Agent) -> str: + try: + from helpers.secrets import get_secrets_manager + from helpers.settings import get_settings_for_prompt + + secrets_manager = get_secrets_manager(agent.context) + secrets = secrets_manager.get_secrets_for_prompt() + variables = get_settings_for_prompt()["variables"] + return agent.read_prompt( + "agent.system.secrets.md", secrets=secrets, vars=variables + ) + except Exception: + return "" diff --git a/extensions/python/system_prompt/_13_skills_prompt.py b/extensions/python/system_prompt/_13_skills_prompt.py index d5de3c4bda412f0bb1ca922db3d9bcb5e75a052c..ba6e6e94d57fdd97e763aef699253bc12678f3ae 100644 --- a/extensions/python/system_prompt/_13_skills_prompt.py +++ b/extensions/python/system_prompt/_13_skills_prompt.py @@ -1,3 +1,40 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:2daedce1206d8117a747cf9ddf1fb13f9bfa9862f932922bdf8dcde2e79045b4 -size 1146 +from typing import Any + +from helpers.extension import Extension, extensible +from helpers import skills as skills_helper, tool_policy +from agent import Agent, LoopData + + +class SkillsPrompt(Extension): + + async def execute( + self, + system_prompt: list[str] = [], + loop_data: LoopData = LoopData(), + **kwargs: Any, + ): + if not self.agent: + return + prompt = await build_prompt(self.agent) + if prompt: + system_prompt.append(prompt) + + +@extensible +async def build_prompt(agent: Agent) -> str: + if not tool_policy.resolve_tool(agent, "skills_tool").allowed: + return "" + + available = skills_helper.list_skills(agent=agent) + result: list[str] = [] + for skill in available: + name = skill.name.strip().replace("\n", " ")[:100] + descr = skill.description.replace("\n", " ").strip() + if len(descr) > 100: + descr = descr[:100].rstrip() + "..." + result.append(f"- {name}: {descr}" if descr else f"- {name}") + + if not result: + return "" + + return agent.read_prompt("agent.system.skills.md", skills="\n".join(result)) diff --git a/extensions/python/system_prompt/_14_project_prompt.py b/extensions/python/system_prompt/_14_project_prompt.py index 26859c9bba54ac0b25d93eec39bf69a5c908bfe1..4afa620cd566a35f7114bd242d7d2c76b5b6d3d8 100644 --- a/extensions/python/system_prompt/_14_project_prompt.py +++ b/extensions/python/system_prompt/_14_project_prompt.py @@ -1,3 +1,48 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:f66abe117baff9351c6247d2b96963fabfaaae7ed53bee46ef8006d108ee3a4c -size 1794 +from typing import Any + +from helpers.extension import Extension, extensible +from helpers import projects +from agent import Agent, LoopData + + +class ProjectPrompt(Extension): + + async def execute( + self, + system_prompt: list[str] = [], + loop_data: LoopData = LoopData(), + **kwargs: Any, + ): + if not self.agent: + return + prompt = await build_prompt(self.agent, loop_data=loop_data) + if prompt: + system_prompt.append(prompt) + + +@extensible +async def build_prompt(agent: Agent, loop_data: LoopData | None = None) -> str: + result = agent.read_prompt("agent.system.projects.main.md") + project_name = agent.context.get_data(projects.CONTEXT_DATA_KEY_PROJECT) + if loop_data: + loop_data.protocol_persistent.pop("agents_md_instructions", None) + loop_data.protocol_persistent.pop("project_instructions", None) + if project_name: + project_vars = projects.build_system_prompt_vars(project_name) + if loop_data and project_vars.get("include_agents_md", True): + agents_md_protocol = projects.build_agents_md_protocol(project_name) + if agents_md_protocol: + loop_data.protocol_persistent["agents_md_instructions"] = ( + agents_md_protocol + ) + if loop_data and project_vars.get("project_instructions"): + loop_data.protocol_persistent["project_instructions"] = agent.read_prompt( + "agent.protocol.projects.instructions.md", + **project_vars, + ) + result += "\n\n" + agent.read_prompt( + "agent.system.projects.active.md", **project_vars + ) + else: + result += "\n\n" + agent.read_prompt("agent.system.projects.inactive.md") + return result diff --git a/extensions/python/tool_execute_after/AGENTS.md b/extensions/python/tool_execute_after/AGENTS.md index 425e27ce45eaa00fb1f6c3e0cb46425bb30db9e0..54c4557c53a91c7020de6ff30e876991789b707c 100644 --- a/extensions/python/tool_execute_after/AGENTS.md +++ b/extensions/python/tool_execute_after/AGENTS.md @@ -1,3 +1,26 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:15fd452d7381a7f7d25d39b294e066ec97df3164debcbe311633a3257979e760 -size 647 +# Tool Execute After Extensions DOX + +## Purpose + +- Own backend processing immediately after tool execution. + +## Ownership + +- Ordered Python files own post-tool secret masking and future tool-result postprocessing. + +## Local Contracts + +- Mask secrets before tool results reach history, UI, or model-visible context. +- Do not alter tool `break_loop` or response semantics unless the hook contract owns that behavior. + +## Work Guidance + +- Coordinate with tool implementations and history hooks when changing tool result data. + +## Verification + +- Smoke-test tool execution with sensitive output after changes. + +## Child DOX Index + +No child DOX files. diff --git a/extensions/python/tool_execute_after/_10_mask_secrets.py b/extensions/python/tool_execute_after/_10_mask_secrets.py index 19eaa439ebffc2a59942d04583b24793a1ed5d8d..362706fd3e4adc742195dd690c51ef85912784ba 100644 --- a/extensions/python/tool_execute_after/_10_mask_secrets.py +++ b/extensions/python/tool_execute_after/_10_mask_secrets.py @@ -1,3 +1,15 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:c37ca95db6bc9c62af50a69a430f111a48d507b49b66a53a80a9eab513a12115 -size 454 +from helpers.extension import Extension +from helpers.secrets import get_secrets_manager +from helpers.tool import Response + + +class MaskToolSecrets(Extension): + + async def execute(self, response: Response | None = None, **kwargs): + if not self.agent: + return + + if not response: + return + secrets_mgr = get_secrets_manager(self.agent.context) + response.message = secrets_mgr.mask_values(response.message) diff --git a/extensions/python/tool_execute_before/AGENTS.md b/extensions/python/tool_execute_before/AGENTS.md index 25b835a27f88a1e9244b1fc7968606476ab2f782..e9f11e78a5dbcad4db2b667ef0a5c77413000dea 100644 --- a/extensions/python/tool_execute_before/AGENTS.md +++ b/extensions/python/tool_execute_before/AGENTS.md @@ -1,3 +1,27 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:c17ba997c1b52d71fb72f518e7f94d020f714661d147ceb38cdc8469751408dc -size 745 +# Tool Execute Before Extensions DOX + +## Purpose + +- Own backend processing immediately before tool execution. + +## Ownership + +- Ordered Python files own prior tool-output replacement, parallel recursion guards, and secret unmasking before execution. + +## Local Contracts + +- Unmask only values required by the target tool. +- Preserve safety checks and do not expose secrets to logs or unrelated tools. +- Keep ordering stable where replacement must occur before unmasking or execution. + +## Work Guidance + +- Coordinate with secret handling, tool argument preparation, and plugin tool gates. + +## Verification + +- Smoke-test tool execution with masked secret arguments and prior-output references after changes. + +## Child DOX Index + +No child DOX files. diff --git a/extensions/python/tool_execute_before/_10_replace_last_tool_output.py b/extensions/python/tool_execute_before/_10_replace_last_tool_output.py index 8b8d53c0c4730693afd1a2d1a9ae9c0737084aa4..c1cdb388353bdd014953c57a2125dc4cfa845330 100644 --- a/extensions/python/tool_execute_before/_10_replace_last_tool_output.py +++ b/extensions/python/tool_execute_before/_10_replace_last_tool_output.py @@ -1,3 +1,36 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:c309d8f1eb104a01dece04e4f63026881abb49bf220387ae46f4598ce2e847ea -size 1272 +from typing import Any +from helpers.extension import Extension + + +class ReplaceLastToolOutput(Extension): + async def execute(self, tool_args: dict[str, Any] | None = None, tool_name: str = "", **kwargs): + if not self.agent: + return + + if not tool_args: + return + + last_call = self.agent.get_data("last_tool_call") or {} + last_output = last_call.get("last_tool_output", "") + if not last_output: + return + + tokens = ("{last_tool_output}", "{{last_tool_output}}") + + def replace_placeholders(value: Any) -> Any: + if isinstance(value, str): + new_val = value + for token in tokens: + new_val = new_val.replace(token, last_output) + return new_val + if isinstance(value, dict): + return {k: replace_placeholders(v) for k, v in value.items()} + if isinstance(value, list): + return [replace_placeholders(v) for v in value] + if isinstance(value, tuple): + return tuple(replace_placeholders(v) for v in value) + return value + + updated_args = replace_placeholders(tool_args) + tool_args.clear() + tool_args.update(updated_args) diff --git a/extensions/python/tool_execute_before/_10_unmask_secrets.py b/extensions/python/tool_execute_before/_10_unmask_secrets.py index 9d991cf6ae8b5a8db1a05e20fce5c24d159a3dbe..3ec649d552d74984b21ecd21646dfaa406ee85dd 100644 --- a/extensions/python/tool_execute_before/_10_unmask_secrets.py +++ b/extensions/python/tool_execute_before/_10_unmask_secrets.py @@ -1,3 +1,21 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:9b8e435be74e6a6d41718fa1de7fe0ff43f5a929da7f0d98e46521f6a8965941 -size 607 +from helpers.extension import Extension +from helpers.secrets import get_secrets_manager + + +class UnmaskToolSecrets(Extension): + + async def execute(self, **kwargs): + if not self.agent: + return + + # Get tool args from kwargs + tool_args = kwargs.get("tool_args") + if not tool_args: + return + + secrets_mgr = get_secrets_manager(self.agent.context) + + # Unmask placeholders in args for actual tool execution + for k, v in tool_args.items(): + if isinstance(v, str): + tool_args[k] = secrets_mgr.replace_placeholders(v) diff --git a/extensions/python/tool_execute_before/_20_block_parallel_recursion.py b/extensions/python/tool_execute_before/_20_block_parallel_recursion.py index b82c1c13047b183f7255ea2eabd9d9b7bdc5c730..a5547ed8ffb496b54ae73e5f803e9d4cfadd5ad8 100644 --- a/extensions/python/tool_execute_before/_20_block_parallel_recursion.py +++ b/extensions/python/tool_execute_before/_20_block_parallel_recursion.py @@ -1,3 +1,15 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:5eff30bc40ccee527217a79ec08403b138440b64c62f48294725eb2875a46959 -size 571 +from helpers.extension import Extension +from helpers.errors import RepairableException +from helpers import parallel_tools + + +class BlockParallelRecursion(Extension): + async def execute(self, tool_name: str = "", **kwargs) -> None: + if tool_name != "parallel": + return + if not parallel_tools.is_parallel_worker(self.agent): + return + raise RepairableException( + "The `parallel` tool cannot be used inside a parallel worker. " + "Finish the current worker task sequentially and return its result." + ) diff --git a/extensions/python/user_message_ui/AGENTS.md b/extensions/python/user_message_ui/AGENTS.md index 7b44fd1bf72e148919a32f845260b7f15d682e6b..2d9a3b70628f94d6caadba8199dfb9f8853ffa51 100644 --- a/extensions/python/user_message_ui/AGENTS.md +++ b/extensions/python/user_message_ui/AGENTS.md @@ -1,3 +1,27 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:e7b9b7a8a1234830652c4cb93ae5b200edcfc99d346dea819761f5299d147a36 -size 690 +# User Message UI Extensions DOX + +## Purpose + +- Own backend behavior triggered around user-visible UI messages. + +## Ownership + +- Ordered Python files own update-check messaging and future user-message UI hooks. + +## Local Contracts + +- Keep proactive UI messages relevant, non-spammy, and safe for display. +- Keep update-available notifications visible until the user dismisses them or opens the updater. +- Do not expose local diagnostics or update data that should stay internal. + +## Work Guidance + +- Gate recurring messages so they do not repeat unnecessarily across chats or tabs. + +## Verification + +- Smoke-test UI message rendering after changes. + +## Child DOX Index + +No child DOX files. diff --git a/extensions/python/user_message_ui/_10_update_check.py b/extensions/python/user_message_ui/_10_update_check.py index bbc428cec58d3059c8d1931b249f9f25b9dba84a..6d26c1d9299d54c994829fd8cd23eeac128ca8b5 100644 --- a/extensions/python/user_message_ui/_10_update_check.py +++ b/extensions/python/user_message_ui/_10_update_check.py @@ -1,3 +1,123 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:52cf96f36419ed2170beeb307a79eb84715557c54645d47861b6eab309af2fd6 -size 4605 +from helpers import notification +from helpers.extension import Extension +from agent import LoopData +from helpers import files, settings, update_check +from helpers.localization import Localization +import datetime +import json + + +# check for newer versions of A0 available and send notification +# check after user message is sent from UI, not API, MCP etc. (user is active and can see the notification) +# do not check too often, use cooldown +# do not notify too often + +last_check = datetime.datetime.fromtimestamp(0, tz=Localization.get().get_tzinfo()) +check_cooldown_seconds = 60 +last_notification_id = None +last_notification_time = datetime.datetime.fromtimestamp(0, tz=Localization.get().get_tzinfo()) +notification_cooldown_seconds = 60 * 60 * 24 +notification_state_file = "usr/update-check-state.json" + + +def _now() -> datetime.datetime: + return Localization.get().now() + + +def _load_notification_state() -> dict: + try: + return json.loads(files.read_file(notification_state_file)) + except Exception: + return {} + + +def _parse_timestamp(value: str | None) -> datetime.datetime | None: + if not value: + return None + try: + parsed = datetime.datetime.fromisoformat(value) + except ValueError: + return None + if parsed.tzinfo: + return parsed.astimezone(Localization.get().get_tzinfo()) + return Localization.get().localize_naive_datetime(parsed) + + +def _remember_notification(notif: dict, now: datetime.datetime): + state = { + "last_notification_at": now.isoformat(), + "last_notification_id": notif.get("id") or "", + "last_notification_group": notif.get("group", "update_check"), + } + files.write_file(notification_state_file, json.dumps(state, indent=2)) + + +class UpdateCheck(Extension): + + async def execute(self, loop_data: LoopData = LoopData(), text: str = "", **kwargs): + if not self.agent: + return + + try: + global last_check, last_notification_id, last_notification_time + + # first check if update check is enabled + current_settings = settings.get_settings() + if not current_settings["update_check_enabled"]: + return + + # check if cooldown has passed + now = _now() + if (now - last_check).total_seconds() < check_cooldown_seconds: + return + last_check = now + + # check for updates + version = await update_check.check_version() + + # if the user should update, send notification + if notif := version.get("notification"): + now = _now() + stored_state = _load_notification_state() + stored_notification_time = _parse_timestamp(stored_state.get("last_notification_at")) + effective_notification_time = stored_notification_time or last_notification_time + + if (now - effective_notification_time).total_seconds() > notification_cooldown_seconds: + last_notification_id = notif.get("id") + last_notification_time = now + try: + _remember_notification(notif, now) + except Exception: + pass + self.send_notification(notif) + except Exception as e: + pass # no need to log if the update server is inaccessible + + + def send_notification(self, notif): + if not self.agent: + return + + message = notif.get( + "message", + "A newer version of Agent Zero is available. Please update to the latest version.", + ) + message = message.replace( + 'Open updater.', + '
' + '
", + ) + notifs = self.agent.context.get_notification_manager() + notifs.send_notification( + title=notif.get("title", "Newer version available"), + message=message, + type=notif.get("type", "info"), + detail=notif.get("detail", ""), + display_time=0, + group=notif.get("group", "update_check"), + priority=notif.get("priority", notification.NotificationPriority.NORMAL), + id=notif.get("id", "update_check_available"), + ) diff --git a/extensions/python/util_model_call_before/AGENTS.md b/extensions/python/util_model_call_before/AGENTS.md index 1256ac4e74dd486fe8070b7dbf76873c38e5d0e7..dc0bd77f8afa7f22dbfc538dad3fffa50d08fd96 100644 --- a/extensions/python/util_model_call_before/AGENTS.md +++ b/extensions/python/util_model_call_before/AGENTS.md @@ -1,3 +1,26 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:3e9ab41869d9300aeac86bda3f8411797178ab54a5fb1b76a27b680605293420 -size 601 +# Utility Model Call Before Extensions DOX + +## Purpose + +- Own preprocessing before utility model calls. + +## Ownership + +- Ordered Python files own secret masking and future utility-call preparation. + +## Local Contracts + +- Mask secrets before utility prompts leave the framework. +- Keep utility model inputs compatible with callers expecting structured outputs. + +## Work Guidance + +- Coordinate masking changes with main model call and error-format masking behavior. + +## Verification + +- Test utility model calls that include masked secret patterns after changes. + +## Child DOX Index + +No child DOX files. diff --git a/extensions/python/util_model_call_before/_10_mask_secrets.py b/extensions/python/util_model_call_before/_10_mask_secrets.py index 422a8c4f30b6b110b6896e60e9284eed4ec82362..60b9545d5d9bbe6b1ace5b784b62e809c0822134 100644 --- a/extensions/python/util_model_call_before/_10_mask_secrets.py +++ b/extensions/python/util_model_call_before/_10_mask_secrets.py @@ -1,3 +1,20 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:d65b2c486d010f8e43c30abd1029a502520a04a899c64a6902b9e373f73d2bbf -size 648 +from helpers.extension import Extension +from helpers.secrets import get_secrets_manager + + +class MaskToolSecrets(Extension): + + async def execute(self, **kwargs): + if not self.agent: + return + + # model call data + call_data:dict = kwargs.get("call_data", {}) + + secrets_mgr = get_secrets_manager(self.agent.context) + + # mask system and user message + if system:=call_data.get("system"): + call_data["system"] = secrets_mgr.mask_values(system) + if message:=call_data.get("message"): + call_data["message"] = secrets_mgr.mask_values(message) \ No newline at end of file diff --git a/extensions/python/webui_ws_connect/AGENTS.md b/extensions/python/webui_ws_connect/AGENTS.md index afbd82b1a021c7c5af1b8528f519f6c08f636855..f722ae7b245a9ef613d2ea2998f55ea26676b69a 100644 --- a/extensions/python/webui_ws_connect/AGENTS.md +++ b/extensions/python/webui_ws_connect/AGENTS.md @@ -1,3 +1,26 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:dbec3eecc8f858d5c5f8c7cff8f1c4dda2b403c1777859e8ce9a184f40e675cc -size 563 +# WebUI WebSocket Connect Extensions DOX + +## Purpose + +- Own backend behavior when a WebUI WebSocket client connects. + +## Ownership + +- Ordered Python files own state-sync behavior for new WebSocket connections. + +## Local Contracts + +- Preserve WebSocket auth/session assumptions. +- Send only state the connected client is allowed to receive. + +## Work Guidance + +- Coordinate connect behavior with frontend WebSocket client and sync store. + +## Verification + +- Smoke-test WebUI connection and initial state sync after changes. + +## Child DOX Index + +No child DOX files. diff --git a/extensions/python/webui_ws_connect/_10_state_sync.py b/extensions/python/webui_ws_connect/_10_state_sync.py index 84ad5bc970b2b509dc72a213d7de4c7e22a01563..f5263221f1347e09c12416cfd7d89b0fc9b427da 100644 --- a/extensions/python/webui_ws_connect/_10_state_sync.py +++ b/extensions/python/webui_ws_connect/_10_state_sync.py @@ -1,3 +1,15 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:53f90d5ce392e8fda645e01036f6f24178925f0def91639e31e6b6ab912df6c6 -size 571 +from helpers.extension import Extension +from helpers.print_style import PrintStyle +from helpers.state_monitor import get_state_monitor, _ws_debug_enabled + + +class StateSync(Extension): + async def execute(self, instance=None, sid: str = "", **kwargs): + if instance is None: + return + + monitor = get_state_monitor() + monitor.bind_manager(instance.manager, handler_id=instance.identifier) + monitor.register_sid(instance.namespace, sid) + if _ws_debug_enabled(): + PrintStyle.debug(f"[WebuiHandler] connect sid={sid}") diff --git a/extensions/python/webui_ws_disconnect/AGENTS.md b/extensions/python/webui_ws_disconnect/AGENTS.md index dadec6cb97a735b77125b0b5f36706a9ac7d62f6..70365289107f7337f19c4201f536e7426404f4db 100644 --- a/extensions/python/webui_ws_disconnect/AGENTS.md +++ b/extensions/python/webui_ws_disconnect/AGENTS.md @@ -1,3 +1,26 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:0191125e588b0356f0d70cd768202834344e54e729bfd8c089fdd9e82a5ac5f7 -size 583 +# WebUI WebSocket Disconnect Extensions DOX + +## Purpose + +- Own backend behavior when a WebUI WebSocket client disconnects. + +## Ownership + +- Ordered Python files own state-sync cleanup for disconnect events. + +## Local Contracts + +- Cleanup must be idempotent and safe for repeated disconnect events. +- Do not remove shared state still needed by other active clients. + +## Work Guidance + +- Coordinate disconnect behavior with frontend reconnect and sync indicators. + +## Verification + +- Smoke-test disconnect and reconnect behavior after changes. + +## Child DOX Index + +No child DOX files. diff --git a/extensions/python/webui_ws_disconnect/_10_state_sync.py b/extensions/python/webui_ws_disconnect/_10_state_sync.py index dc8192f21f7bb3947f078f7524b9cc6b464022d9..adde8fdabef956e1dab7d91b5fff36e34a23d0b8 100644 --- a/extensions/python/webui_ws_disconnect/_10_state_sync.py +++ b/extensions/python/webui_ws_disconnect/_10_state_sync.py @@ -1,3 +1,13 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:ab4d45092c4cb8357b32ec7b542df249e966ead38bacb6c49fd4995d98c420e3 -size 471 +from helpers.extension import Extension +from helpers.print_style import PrintStyle +from helpers.state_monitor import get_state_monitor, _ws_debug_enabled + + +class StateSync(Extension): + async def execute(self, instance=None, sid: str = "", **kwargs): + if instance is None: + return + + get_state_monitor().unregister_sid(instance.namespace, sid) + if _ws_debug_enabled(): + PrintStyle.debug(f"[WebuiHandler] disconnect sid={sid}") diff --git a/extensions/python/webui_ws_event/AGENTS.md b/extensions/python/webui_ws_event/AGENTS.md index 1cf591ce497cd96e9407397db9ba6f53a508a0bd..a47e147b23bd375895fbc633eeac7a881d007a13 100644 --- a/extensions/python/webui_ws_event/AGENTS.md +++ b/extensions/python/webui_ws_event/AGENTS.md @@ -1,3 +1,26 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:a890a37f19c55cffd2efa2db7055e1a8d4fe62388e98c958bbf39e1e88cf2931 -size 566 +# WebUI WebSocket Event Extensions DOX + +## Purpose + +- Own backend behavior for incoming WebUI WebSocket events. + +## Ownership + +- Ordered Python files own state-sync event handling and future WebSocket event extensions. + +## Local Contracts + +- Validate event names and payloads before acting on them. +- Preserve auth/session boundaries for all WebSocket events. + +## Work Guidance + +- Coordinate event changes with frontend WebSocket client and sync store. + +## Verification + +- Smoke-test relevant WebSocket events after changes. + +## Child DOX Index + +No child DOX files. diff --git a/extensions/python/webui_ws_event/_10_state_sync.py b/extensions/python/webui_ws_event/_10_state_sync.py index cf6591c219e06213048f5e60c6698def2fc56a13..c29f1c0d86bae5769ad2d0effea70fbe2bcf83b9 100644 --- a/extensions/python/webui_ws_event/_10_state_sync.py +++ b/extensions/python/webui_ws_event/_10_state_sync.py @@ -1,3 +1,66 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:7230bdffdffa2013dc86011be8eea0eebb44dabc879db4d4796b94ffbdfebf87 -size 2175 +from helpers import runtime +from helpers.extension import Extension +from helpers.print_style import PrintStyle +from helpers.state_monitor import get_state_monitor, _ws_debug_enabled +from helpers.state_snapshot import ( + StateRequestValidationError, + parse_state_request_payload, +) + + +class StateSync(Extension): + async def execute( + self, + instance=None, + sid: str = "", + event_type: str = "", + data: dict | None = None, + response_data: dict | None = None, + **kwargs, + ): + if instance is None or data is None: + return + + if event_type != "state_request": + return + + correlation_id = data.get("correlationId") + try: + request = parse_state_request_payload(data) + except StateRequestValidationError as exc: + PrintStyle.warning( + f"[WebuiHandler] INVALID_REQUEST sid={sid} reason={exc.reason} details={exc.details!r}" + ) + if response_data is not None: + response_data["code"] = "INVALID_REQUEST" + response_data["message"] = str(exc) + return + + if _ws_debug_enabled(): + PrintStyle.debug( + f"[WebuiHandler] state_request sid={sid} context={request.context!r} " + f"log_from={request.log_from} notifications_from={request.notifications_from} timezone={request.timezone!r} " + f"correlation_id={correlation_id}" + ) + + seq_base = 1 + monitor = get_state_monitor() + monitor.update_projection( + instance.namespace, + sid, + request=request, + seq_base=seq_base, + ) + monitor.mark_dirty( + instance.namespace, + sid, + reason="webui_ws_event.StateSync.state_request", + ) + if _ws_debug_enabled(): + PrintStyle.debug( + f"[WebuiHandler] state_request accepted sid={sid} seq_base={seq_base}" + ) + + if response_data is not None: + response_data["runtime_epoch"] = runtime.get_runtime_id() + response_data["seq_base"] = seq_base diff --git a/extensions/webui/AGENTS.md b/extensions/webui/AGENTS.md index 0daec0afc7fbf6d9600fdd3dcbd29563d1a5aa0f..f2c530b9070c9fa1f96f82167e93c826bcffce8a 100644 --- a/extensions/webui/AGENTS.md +++ b/extensions/webui/AGENTS.md @@ -1,3 +1,48 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:eca453794f0b7b224c804fc8a595132e1fb7c1ff0817c4f8f1d4cb043144b5ba -size 2608 +# WebUI Extensions DOX + +## Purpose + +- Own built-in frontend extension contributions under `extensions/webui/`. +- Keep WebUI extension points compatible with the core loader and plugin extension model. + +## Ownership + +- Each direct subdirectory is one frontend extension point. +- `.html` files are injected as component references through ``. +- `.js` and `.mjs` files export default functions called by `callJsExtensions`. + +## Local Contracts + +- HTML contributions must be valid component fragments and include Alpine state only where needed. +- JavaScript extension modules must export a default function. +- Extension code must not assume a plugin is installed unless it guards that dependency. +- Keep extension point names synchronized with `x-extension` IDs and `callJsExtensions()` callers. + +## Work Guidance + +- Prefer small extension modules that delegate to existing WebUI stores or helpers. +- Use the notification store for user-facing success, warning, or error feedback. +- Avoid global DOM queries when an extension hook provides scoped nodes or context. + +## Verification + +- Manually load the WebUI or run targeted frontend/WebUI tests after changing visible extension behavior. +- Verify extension cache clearing paths when adding new extension points. + +## Child DOX Index + +Direct child DOX files: + +| Child | Scope | +| --- | --- | +| [fetch_api_call_after/AGENTS.md](fetch_api_call_after/AGENTS.md) | Frontend hooks after raw `fetchApi()` calls. | +| [fetch_api_call_before/AGENTS.md](fetch_api_call_before/AGENTS.md) | Frontend hooks before raw `fetchApi()` calls. | +| [get_message_handler/AGENTS.md](get_message_handler/AGENTS.md) | Message rendering handler extensions. | +| [initFw_end/AGENTS.md](initFw_end/AGENTS.md) | Post-WebUI-framework-initialization extensions. | +| [json_api_call_after/AGENTS.md](json_api_call_after/AGENTS.md) | Frontend hooks after `callJsonApi()` calls. | +| [json_api_call_before/AGENTS.md](json_api_call_before/AGENTS.md) | Frontend hooks before `callJsonApi()` calls. | +| [right-canvas-panels/AGENTS.md](right-canvas-panels/AGENTS.md) | Built-in right-canvas panel HTML contributions. | +| [right_canvas_register_surfaces/AGENTS.md](right_canvas_register_surfaces/AGENTS.md) | Built-in right-canvas surface registrations. | +| [set_messages_after_loop/AGENTS.md](set_messages_after_loop/AGENTS.md) | Frontend hooks after message DOM updates. | +| [set_messages_before_loop/AGENTS.md](set_messages_before_loop/AGENTS.md) | Frontend hooks before message DOM updates. | +| [webui_ws_push/AGENTS.md](webui_ws_push/AGENTS.md) | WebUI WebSocket push-event behavior. | diff --git a/extensions/webui/fetch_api_call_after/AGENTS.md b/extensions/webui/fetch_api_call_after/AGENTS.md index d79cb354390bddfef8c1464adb7b0ad00c4b5e64..963be5f11bf0b9c1280f1b8a1c886780bc417baa 100644 --- a/extensions/webui/fetch_api_call_after/AGENTS.md +++ b/extensions/webui/fetch_api_call_after/AGENTS.md @@ -1,3 +1,26 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:3e9e591ebd3fe47fb36a4171bf7a292b89520d2f10f83864be2f92e652447cf5 -size 634 +# Fetch API Call After Extensions DOX + +## Purpose + +- Own frontend extension hooks that run after raw `fetchApi()` calls. + +## Ownership + +- Files in this folder own after-call behavior for CSRF-aware raw fetch flows. + +## Local Contracts + +- JavaScript modules must export a default function when present. +- Do not consume response bodies unless the hook contract explicitly passes a clone or mutable context for that purpose. + +## Work Guidance + +- Keep after-call extensions lightweight and safe for all fetch callers. + +## Verification + +- Smoke-test frontend API calls after adding behavior here. + +## Child DOX Index + +No child DOX files. diff --git a/extensions/webui/fetch_api_call_before/AGENTS.md b/extensions/webui/fetch_api_call_before/AGENTS.md index ab820d8be54ceb69a03a679d5218918496f3a61f..c715757d5930262561a40918e121203efa3f8c38 100644 --- a/extensions/webui/fetch_api_call_before/AGENTS.md +++ b/extensions/webui/fetch_api_call_before/AGENTS.md @@ -1,3 +1,26 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:d332098957e0730486ba06988a990d470c6bcfaae9ae05a6975ae444b90db12e -size 576 +# Fetch API Call Before Extensions DOX + +## Purpose + +- Own frontend extension hooks that run before raw `fetchApi()` calls. + +## Ownership + +- Files in this folder own before-call behavior for CSRF-aware raw fetch flows. + +## Local Contracts + +- JavaScript modules must export a default function when present. +- Preserve CSRF, auth, and redirect behavior owned by `/js/api.js`. + +## Work Guidance + +- Avoid broad request mutation that surprises unrelated API callers. + +## Verification + +- Smoke-test affected frontend API calls after changes. + +## Child DOX Index + +No child DOX files. diff --git a/extensions/webui/get_message_handler/AGENTS.md b/extensions/webui/get_message_handler/AGENTS.md index 7f6eb9c7350fb027dcf0e3fbfe4cab8301e59897..ea0a36977842441fa23d20b318a925e2b6024522 100644 --- a/extensions/webui/get_message_handler/AGENTS.md +++ b/extensions/webui/get_message_handler/AGENTS.md @@ -1,3 +1,27 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:553d954d363ae9caac2616d502ef327625a61b57aef264edf6a44870d429e215 -size 664 +# Get Message Handler Extensions DOX + +## Purpose + +- Own frontend extensions that provide or modify message rendering handlers. + +## Ownership + +- Files in this folder own handler registration behavior for rendered chat messages. + +## Local Contracts + +- JavaScript modules must export a default function when present. +- Preserve mutable context contracts used by `/js/messages.js`. +- Do not render unsanitized model or user content. + +## Work Guidance + +- Coordinate handler changes with message components and plugin message extensions. + +## Verification + +- Smoke-test message rendering for affected message types after changes. + +## Child DOX Index + +No child DOX files. diff --git a/extensions/webui/initFw_end/AGENTS.md b/extensions/webui/initFw_end/AGENTS.md index 499b609b3eaa5dda8f73132bd131db9174a08835..7390427ded94cd6285fc52d2c4d1b07d8fd1de73 100644 --- a/extensions/webui/initFw_end/AGENTS.md +++ b/extensions/webui/initFw_end/AGENTS.md @@ -1,3 +1,27 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:b1a11494a4e3f6545379a24c43cf1a30ab7ea68360ecb54fa175f667e4940643 -size 670 +# Init Framework End Extensions DOX + +## Purpose + +- Own frontend extensions that run after WebUI framework initialization. + +## Ownership + +- JavaScript files own post-bootstrap global setup such as self-update helpers and session-scoped UI restoration hooks. + +## Local Contracts + +- JavaScript modules must export a default function. +- Setup must be idempotent across reloads and cache resets. +- Do not register duplicate global listeners. + +## Work Guidance + +- Coordinate initialization changes with `/js/initFw.js` and component lifecycle directives. + +## Verification + +- Smoke-test WebUI startup and browser console after changes. + +## Child DOX Index + +No child DOX files. diff --git a/extensions/webui/initFw_end/restoreRestorableModals.js b/extensions/webui/initFw_end/restoreRestorableModals.js index 07edb2e8ce486d5a0a77f742773f3e996b86dd00..51462c2bfcbd03366759921ce1a5bd258c2b86b4 100644 --- a/extensions/webui/initFw_end/restoreRestorableModals.js +++ b/extensions/webui/initFw_end/restoreRestorableModals.js @@ -1,3 +1,5 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:f756412bce11ea7a07758c7a79ac875a69adb977b34d1e8e0bb3cb3f58a07d87 -size 149 +import { restoreRestorableModalStack } from "/js/modals.js"; + +export default function restoreRestorableModals() { + restoreRestorableModalStack(); +} diff --git a/extensions/webui/initFw_end/selfUpdateGlobal.js b/extensions/webui/initFw_end/selfUpdateGlobal.js index 46670b04f5f1e6ffa8bbead01e1c90d948b6d5bf..b86b50aafc91df048540a186a7385aa96f77822e 100644 --- a/extensions/webui/initFw_end/selfUpdateGlobal.js +++ b/extensions/webui/initFw_end/selfUpdateGlobal.js @@ -1,3 +1,5 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:d7ce3bb10568e85774f884354afd74ea33f2f17958d76339557552a83f847acd -size 171 +import { store } from "/components/settings/external/self-update-store.js"; + +export default async function selfUpdateGlobal(ctx) { + // do nothing, the import is enough +} diff --git a/extensions/webui/json_api_call_after/AGENTS.md b/extensions/webui/json_api_call_after/AGENTS.md index 47d8241bcaa334d377037d5bc0b862d5557313b8..96f356b8d2b010c8078f4e006e934c1aa6ad285c 100644 --- a/extensions/webui/json_api_call_after/AGENTS.md +++ b/extensions/webui/json_api_call_after/AGENTS.md @@ -1,3 +1,26 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:b42c8b00b2d581e6cf4736824858c19636208c457c46f9729341675324f0d1a1 -size 585 +# JSON API Call After Extensions DOX + +## Purpose + +- Own frontend extension hooks that run after `callJsonApi()` calls. + +## Ownership + +- JavaScript files own after-call behavior such as cache reset handling. + +## Local Contracts + +- JavaScript modules must export a default function. +- Preserve JSON API response contracts and avoid hiding errors from callers. + +## Work Guidance + +- Keep global side effects narrow and tied to explicit API results or mutable contexts. + +## Verification + +- Smoke-test JSON API callers affected by extension changes. + +## Child DOX Index + +No child DOX files. diff --git a/extensions/webui/json_api_call_after/cache_reset.js b/extensions/webui/json_api_call_after/cache_reset.js index be6962ce563ccc8b2174d040a2b87381af15b729..b09e96bbce41404e88611eaee35cd88c7706525c 100644 --- a/extensions/webui/json_api_call_after/cache_reset.js +++ b/extensions/webui/json_api_call_after/cache_reset.js @@ -1,3 +1,14 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:ef416a6db9481288f7d4f861cfdedfeffb228888a5d965af0e4c8404dae0cf66 -size 332 +import { clear } from "/js/cache.js"; + +export default async function resetCache(ctx) { + try { + // clear frontend cache areas when backend caches are cleared via API + if (ctx.endpoint == "cache_reset") { + for (const area of ctx.data.areas) { + clear(area); + } + } + } catch (e) { + console.error(e); + } +} diff --git a/extensions/webui/json_api_call_before/AGENTS.md b/extensions/webui/json_api_call_before/AGENTS.md index 9fec86a2554d64d91b55cdf07176e526f04443ab..b14f04ea9a3b69c96e1f3618a2dc8e56929d1c8d 100644 --- a/extensions/webui/json_api_call_before/AGENTS.md +++ b/extensions/webui/json_api_call_before/AGENTS.md @@ -1,3 +1,26 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:9598aed7f6ad9604983591d8467c4967038b6579a9bd01b5ee2e5525294090cb -size 583 +# JSON API Call Before Extensions DOX + +## Purpose + +- Own frontend extension hooks that run before `callJsonApi()` calls. + +## Ownership + +- Files in this folder own before-call behavior for JSON API requests. + +## Local Contracts + +- JavaScript modules must export a default function when present. +- Preserve CSRF/auth behavior and JSON payload shape expected by `/js/api.js`. + +## Work Guidance + +- Avoid broad request mutation that affects unrelated plugin or core API calls. + +## Verification + +- Smoke-test affected JSON API calls after changes. + +## Child DOX Index + +No child DOX files. diff --git a/extensions/webui/right-canvas-panels/AGENTS.md b/extensions/webui/right-canvas-panels/AGENTS.md index 1bc5ed7b13fc880b3b558e740b34bd42d02d3149..014adda270d90dd9c87c54c7cef1c6dd63955e50 100644 --- a/extensions/webui/right-canvas-panels/AGENTS.md +++ b/extensions/webui/right-canvas-panels/AGENTS.md @@ -1,3 +1,28 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:58cd137985a9d03bc535dcc24df6d2be5a278b5f8a3788cfac37c54bc1d26729 -size 824 +# Right Canvas Panel Extensions DOX + +## Purpose + +- Own built-in HTML panel contributions for the right-canvas surface area. + +## Ownership + +- `.html` files mount WebUI components into the `right-canvas-panels` extension point. +- Panel wrappers own `data-surface-id` anchors and active/mounted visibility bindings. + +## Local Contracts + +- Each panel must correspond to a registered right-canvas surface ID. +- Use `` for reusable component content instead of duplicating panel implementations. +- Keep canvas panels compatible with `.right-canvas-surface-panel` layout semantics. + +## Work Guidance + +- Prefer thin wrappers that delegate lifecycle and state to the owning component store. + +## Verification + +- Smoke-test opening the matching surface from the right-canvas rail. + +## Child DOX Index + +No child DOX files. diff --git a/extensions/webui/right-canvas-panels/files-panel.html b/extensions/webui/right-canvas-panels/files-panel.html index 034014c362cfbc5ec041feb5a48c933cc6a99253..fc0e7a1e637d81891e20acc997a210bee4027b3a 100644 --- a/extensions/webui/right-canvas-panels/files-panel.html +++ b/extensions/webui/right-canvas-panels/files-panel.html @@ -1,3 +1,11 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:9db7bbd2f7edc21ec6aabfbae5226eeaaab4bdd25bcee0c1e427b4cf75b145c5 -size 410 +
+ +
diff --git a/extensions/webui/right_canvas_register_surfaces/AGENTS.md b/extensions/webui/right_canvas_register_surfaces/AGENTS.md index e06ea892f6dd6940a0852d1697bd5990baa55bce..1387cda749837311c9f19bf54eb762a9cf1cf3b5 100644 --- a/extensions/webui/right_canvas_register_surfaces/AGENTS.md +++ b/extensions/webui/right_canvas_register_surfaces/AGENTS.md @@ -1,3 +1,27 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:5a8667b504ade89d0f623a5648ab36c1bb857b6cb32b3261b4d904479ebc075d -size 674 +# Right Canvas Surface Extensions DOX + +## Purpose + +- Own frontend registration of built-in right-canvas surfaces. + +## Ownership + +- JavaScript files own registration of remote link, space agent, file browser, and future core canvas surfaces. + +## Local Contracts + +- JavaScript modules must export a default function. +- Surface IDs must be unique and stable. +- Registered surfaces must point to valid components or handlers. + +## Work Guidance + +- Coordinate surface registration changes with `webui/components/canvas/` and related plugin panels. + +## Verification + +- Smoke-test opening each registered right-canvas surface after changes. + +## Child DOX Index + +No child DOX files. diff --git a/extensions/webui/right_canvas_register_surfaces/register-files.js b/extensions/webui/right_canvas_register_surfaces/register-files.js index bec7156ecf1afeb01366257838f02e058a07088f..f1cd6bd40b1670cbadf9636a14174fb369be9e40 100644 --- a/extensions/webui/right_canvas_register_surfaces/register-files.js +++ b/extensions/webui/right_canvas_register_surfaces/register-files.js @@ -1,3 +1,44 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:c7e92622aee4b824f63d74c5926e1aac3e31a0ed3df42e3d96c43415eb155c53 -size 1542 +import { store as fileBrowserStore } from "/components/modals/file-browser/file-browser-store.js"; + +function waitForElement(selector, timeoutMs = 3000) { + const found = document.querySelector(selector); + if (found) return Promise.resolve(found); + return new Promise((resolve) => { + const timeout = globalThis.setTimeout(() => { + observer.disconnect(); + resolve(document.querySelector(selector)); + }, timeoutMs); + const observer = new MutationObserver(() => { + const element = document.querySelector(selector); + if (!element) return; + globalThis.clearTimeout(timeout); + observer.disconnect(); + resolve(element); + }); + observer.observe(document.body, { childList: true, subtree: true }); + }); +} + +export default async function registerFilesSurface(surfaces) { + surfaces.registerSurface({ + id: "files", + title: "Files", + icon: "folder", + order: 5, + modalPath: "modals/file-browser/file-browser.html", + beginDockHandoff() { + fileBrowserStore.beginSurfaceHandoff?.(); + }, + finishDockHandoff(payload = {}) { + fileBrowserStore.finishSurfaceHandoff?.(payload); + }, + cancelDockHandoff() { + fileBrowserStore.cancelSurfaceHandoff?.(); + }, + async open(payload = {}) { + const panel = await waitForElement('[data-surface-id="files"] .file-browser-root'); + if (!panel) throw new Error("Files surface panel did not mount."); + await fileBrowserStore.openSurface(payload.path || payload.filePath || payload.directory || ""); + }, + }); +} diff --git a/extensions/webui/right_canvas_register_surfaces/register-remote-link.js b/extensions/webui/right_canvas_register_surfaces/register-remote-link.js index bb1f17c65be56d43c708ccfea4a87a7c4726e3fd..dd0d95d4d1afe79d5a191b1e9934ef82c54aefa2 100644 --- a/extensions/webui/right_canvas_register_surfaces/register-remote-link.js +++ b/extensions/webui/right_canvas_register_surfaces/register-remote-link.js @@ -1,3 +1,3 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:6425c432f7222030d3eb14daa437967c66ffeb299f05b18c83813f2e7176a04d -size 145 +export default async function registerRemoteLinkAction() { + // Remote Control is opened from the sidebar dropdown, not the right canvas rail. +} diff --git a/extensions/webui/right_canvas_register_surfaces/register-space-agent.js b/extensions/webui/right_canvas_register_surfaces/register-space-agent.js index 201c12c0025272fcdaba452f92b6ef9b2826fdea..5fccb5ecc42c46e49358f23145eb267c81641f97 100644 --- a/extensions/webui/right_canvas_register_surfaces/register-space-agent.js +++ b/extensions/webui/right_canvas_register_surfaces/register-space-agent.js @@ -1,3 +1,3 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:8ced6114cbbae8fe627342c8b2c250272cae24d45887fba775a8d04ce0edbc1e -size 142 +export default async function registerSpaceAgentAction() { + // Space Agent is opened from the sidebar dropdown, not the right canvas rail. +} diff --git a/extensions/webui/set_messages_after_loop/AGENTS.md b/extensions/webui/set_messages_after_loop/AGENTS.md index 1f1523afc8fb6e8ecd762d625f587666a0229fce..85fec404413c59c80995e08aee24b11ef7a22ac4 100644 --- a/extensions/webui/set_messages_after_loop/AGENTS.md +++ b/extensions/webui/set_messages_after_loop/AGENTS.md @@ -1,3 +1,27 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:9a13f9ee44f505a75d9ff6a2bcd40cfbfaf7f6f0c3cf91fb024d76ffc1a5b32d -size 820 +# Set Messages After Loop Extensions DOX + +## Purpose + +- Own frontend extensions that run after message DOM updates complete. + +## Ownership + +- Files in this folder own after-render message behavior. + +## Local Contracts + +- JavaScript modules must export a default function when present. +- Preserve message DOM stability and avoid duplicate controls on repeated renders. +- Offscreen live entries in a virtualized chat may appear in `context.results` with `result.virtualized === true` and `result.element === null`; DOM extensions must guard `element`, while args-only side effects may still run. + +## Work Guidance + +- Use stable markers when injecting controls into message elements. + +## Verification + +- Smoke-test message rerendering and extension-injected controls after changes. + +## Child DOX Index + +No child DOX files. diff --git a/extensions/webui/set_messages_before_loop/AGENTS.md b/extensions/webui/set_messages_before_loop/AGENTS.md index bdbe4af40ddb9a902beeb235c281db83197d5725..d417ba566a97b7c27217c633345f977fe4549cce 100644 --- a/extensions/webui/set_messages_before_loop/AGENTS.md +++ b/extensions/webui/set_messages_before_loop/AGENTS.md @@ -1,3 +1,26 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:d65c552fe70aaa356f9dc16ceacb0f7b0150891a1828d9ecb17322579480798e -size 568 +# Set Messages Before Loop Extensions DOX + +## Purpose + +- Own frontend extensions that run before message DOM updates. + +## Ownership + +- Files in this folder own pre-render message behavior. + +## Local Contracts + +- JavaScript modules must export a default function when present. +- Do not remove DOM state needed by message rendering unless the mutable context owns it. + +## Work Guidance + +- Coordinate pre-render behavior with `/js/messages.js` and message components. + +## Verification + +- Smoke-test message updates after changes. + +## Child DOX Index + +No child DOX files. diff --git a/extensions/webui/webui_ws_push/AGENTS.md b/extensions/webui/webui_ws_push/AGENTS.md index 3ef99e937484b020af4281c981900e89e822cb24..58d838730f700a1812a500cc7789b22df6fcb358 100644 --- a/extensions/webui/webui_ws_push/AGENTS.md +++ b/extensions/webui/webui_ws_push/AGENTS.md @@ -1,3 +1,27 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:bebc9a8d149e1e5a2be5bcfeb054e0ff79651a95059a28548061c269955a9e16 -size 594 +# WebUI WebSocket Push Extensions DOX + +## Purpose + +- Own frontend behavior for WebUI WebSocket push events. + +## Ownership + +- JavaScript files own push-event side effects such as cache clearing. + +## Local Contracts + +- JavaScript modules must export a default function. +- Validate event payload shape before acting. +- Keep cache or state resets scoped to the event type. + +## Work Guidance + +- Coordinate push behavior with backend WebSocket event extensions and frontend stores. + +## Verification + +- Smoke-test relevant WebSocket push events after changes. + +## Child DOX Index + +No child DOX files. diff --git a/extensions/webui/webui_ws_push/clear_cache.js b/extensions/webui/webui_ws_push/clear_cache.js index 99e535692c06fdcaf10fa0c45075886e6e5fdbcc..8ee5491f82379ed6c7d3173e6b8f986b4f66bb6f 100644 --- a/extensions/webui/webui_ws_push/clear_cache.js +++ b/extensions/webui/webui_ws_push/clear_cache.js @@ -1,3 +1,21 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:cd457c882cd97164cf472db1b15810ae94341da445e54262f4395ee736094f04 -size 549 +import { clear, clear_all } from "/js/cache.js"; + +export default async function clearCache(eventType, envelope) { + try { + // clear frontend cache areas when backend caches are cleared via API + if (eventType == "clear_cache") { + const areas = envelope?.data?.areas || []; + console.log("Clearing caches", areas); + if (areas.length > 0) { + for (const area of areas) { + clear(area); + } + } else { + // clear all caches + clear_all(); + } + } + } catch (e) { + console.error(e); + } +} diff --git a/helpers/AGENTS.md b/helpers/AGENTS.md index ea3c73699010f244bcfef002a42dedb1d9c28b1a..39d24d71ede0ea4a9c184cb601aa87841bc78e46 100644 --- a/helpers/AGENTS.md +++ b/helpers/AGENTS.md @@ -1,3 +1,41 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:2cd144169cad81a2d7bc14ede21cfdf6f92a314a03617c56c261e2a89ff4913e -size 2831 +# Backend Helpers DOX + +## Purpose + +- Own shared Python framework utilities used by agents, APIs, tools, plugins, WebSockets, persistence, and runtime services. +- Keep cross-cutting behavior stable and tested. + +## Ownership + +- Helper modules provide reusable services; feature-specific route handlers belong in `api/`, tool behavior in `tools/`, and plugin-local logic inside plugin directories. +- Security, auth, settings, file access, plugin discovery, extension dispatch, notifications, state snapshots, scheduler, tunnel, and WebSocket primitives live here. + +## Local Contracts + +- Preserve public helper APIs used by core code and plugins unless all callers, docs, and tests are updated. +- Use structured parsers and serializers for YAML, JSON, paths, and URLs instead of ad hoc string handling. +- Keep path handling constrained to intended roots for user files, uploads, downloads, projects, and workdirs. +- Project metadata defaults must remain backwards-compatible; missing `include_agents_md` is treated as enabled, project instruction file content is injected with an explicit source path, and active-project AGENTS.md path-chain guidance is assembled into prompt protocol without duplicating the project root AGENTS.md. +- Do not hardcode secrets, provider keys, local absolute paths, or environment-specific values. +- Use `RepairableException` for errors an agent may be able to fix. +- This directory is a file-documented DOX profile: every direct `*.py` helper module must have a same-directory `*.py.dox.md` file named by appending `.dox.md` to the full Python filename. +- The `*.py.dox.md` file owns helper purpose, public classes/functions, cross-module contracts, persistence or side effects, path/security assumptions, important dependencies, and verification guidance. +- When a helper module is added, removed, renamed, or behaviorally changed, update its matching `*.py.dox.md` in the same change. +- Do not leave stale file-level DOX after helper deletion or rename. + +## Work Guidance + +- Prefer cohesive helper modules over adding unrelated utilities to large files. +- Keep imports acyclic where possible; defer imports inside functions only when needed to avoid startup cycles. +- For changes touching auth, CSRF, files, plugins, tunnels, WebSockets, or model calls, read the caller and tests before editing. +- During the DOX pass, verify that every direct `*.py` file has a matching `*.py.dox.md` and that changed helper behavior is described there. + +## Verification + +- Run targeted tests for changed helper modules. +- Run security regression tests for auth, CSRF, filesystem, WebSocket, tunnel, upload, or image-serving changes. +- Check file-level documentation coverage with a script or shell loop that verifies each `helpers/*.py` has a matching `helpers/*.py.dox.md`. + +## Child DOX Index + +No child DOX files. diff --git a/helpers/api.py b/helpers/api.py index 1da10cb4ba0b90ebbf7f064bc7a533e456c95b98..6047f3eb9acb9bbeddf12a49133b499fe0f5c1bf 100644 --- a/helpers/api.py +++ b/helpers/api.py @@ -1,3 +1,302 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:f88f0dae9b2c4405f56893bfad9efe2c6965717d6f4ce3755ffb0e6318542b52 -size 10096 +from abc import abstractmethod +import json +import threading +from urllib.parse import urlsplit, unquote +from functools import wraps +from pathlib import Path +from typing import Union, Dict, Any +from flask import ( + Request, + Response, + jsonify, + Flask, + session, + request, + send_file, + redirect, + url_for, +) +from werkzeug.wrappers.response import Response as BaseResponse +from helpers.print_style import PrintStyle +from helpers.errors import format_error +from helpers import files, cache + +ThreadLockType = Union[threading.Lock, threading.RLock] + +CACHE_AREA = "api_handlers(api)" +# cache.toggle_area(CACHE_AREA, False) # cache off for now + +Input = dict +Output = Union[Dict[str, Any], Response] + + +class ApiHandler: + def __init__(self, app: Flask, thread_lock: ThreadLockType): + self.app = app + self.thread_lock = thread_lock + + @classmethod + def requires_loopback(cls) -> bool: + return False + + @classmethod + def requires_api_key(cls) -> bool: + return False + + @classmethod + def requires_auth(cls) -> bool: + return True + + @classmethod + def get_methods(cls) -> list[str]: + return ["POST"] + + @classmethod + def requires_csrf(cls) -> bool: + return cls.requires_auth() + + @abstractmethod + async def process(self, input: Input, request: Request) -> Output: + pass + + async def handle_request(self, request: Request) -> Response: + try: + # input data from request based on type + input_data: Input = {} + if request.is_json: + try: + if request.data: # Check if there's any data + input_data = request.get_json() + # If empty or not valid JSON, use empty dict + except Exception as e: + # Just log the error and continue with empty input + PrintStyle().print(f"Error parsing JSON: {str(e)}") + input_data = {} + else: + # input_data = {"data": request.get_data(as_text=True)} + input_data = {} + + # process via handler + output = await self.process(input_data, request) + + # return output based on type + if isinstance(output, Response): + return output + else: + response_json = json.dumps(output) + return Response( + response=response_json, status=200, mimetype="application/json" + ) + + # return exceptions with 500 + except Exception as e: + error = format_error(e) + PrintStyle.error(f"API error: {error}") + return Response(response=error, status=500, mimetype="text/plain") + + # get context to run agent zero in + def use_context(self, ctxid: str, create_if_not_exists: bool = True): + from helpers.context_utils import use_context as _use_context + return _use_context(self.thread_lock, ctxid, create_if_not_exists) + + +from helpers.network import is_loopback_address + + +def is_safe_next_url(value: str | None) -> bool: + """Return True when value is a safe same-origin redirect target.""" + if not value: + return False + if "\r" in value or "\n" in value: + return False + # Reject raw backslashes (browsers normalize `/\host` to `//host` -> external). + if "\\" in value: + return False + + # Decode percent-escapes so encoded backslashes (e.g. `%5C`) are caught too. + decoded = unquote(value) + if "\\" in decoded: + return False + + parsed = urlsplit(decoded) + if parsed.scheme or parsed.netloc: + return False + + # Require an absolute path within this origin, but reject protocol-relative URLs. + return parsed.path.startswith("/") and not parsed.path.startswith("//") + + +def get_safe_next_url(value: str | None, fallback: str | None = None) -> str | None: + """Return value if it is a safe next URL, otherwise return a safe fallback.""" + if is_safe_next_url(value): + return value + if is_safe_next_url(fallback): + return fallback + return None + + +def get_current_request_next_url() -> str: + """Return the current request path/query as a safe relative redirect target.""" + next_url = request.full_path if request.query_string else request.path + return get_safe_next_url(next_url, url_for("serve_index")) or url_for("serve_index") + + +def requires_api_key(f): + @wraps(f) + async def decorated(*args, **kwargs): + from helpers.settings import get_settings + + valid_api_key = get_settings()["mcp_server_token"] + + if api_key := request.headers.get("X-API-KEY"): + if api_key != valid_api_key: + return Response("Invalid API key", 401) + elif request.json and request.json.get("api_key"): + api_key = request.json.get("api_key") + if api_key != valid_api_key: + return Response("Invalid API key", 401) + else: + return Response("API key required", 401) + return await f(*args, **kwargs) + + return decorated + + +def requires_loopback(f): + @wraps(f) + async def decorated(*args, **kwargs): + if not is_loopback_address(str(request.remote_addr)): + return Response("Access denied.", 403, {}) + return await f(*args, **kwargs) + + return decorated + + +def requires_auth(f): + @wraps(f) + async def decorated(*args, **kwargs): + from helpers import login + + user_pass_hash = login.get_credentials_hash() + if not user_pass_hash: + return await f(*args, **kwargs) + if session.get("authentication") != user_pass_hash: + return redirect(url_for("login_handler", next=get_current_request_next_url())) + return await f(*args, **kwargs) + + return decorated + + +def csrf_protect(f): + @wraps(f) + async def decorated(*args, **kwargs): + from helpers import runtime + + token = session.get("csrf_token") + header = request.headers.get("X-CSRF-Token") + cookie = request.cookies.get("csrf_token_" + runtime.get_runtime_id()) + sent = header or cookie + if not token or not sent or token != sent: + return Response("CSRF token missing or invalid", 403) + return await f(*args, **kwargs) + + return decorated + + +def register_api_route(app: Flask, lock: ThreadLockType) -> None: + from helpers.modules import load_classes_from_file + from helpers import plugins + + async def _dispatch(path: str) -> BaseResponse: + # Return cached wrapped handler if available + cached = cache.get(CACHE_AREA, path) + if cached is not None: + return await cached() + + # Resolve file path for the handler + # Try built-in and plugin api folders before the user fallback + handler_cls: type[ApiHandler] | None = None + + # Check built-in python/api/.py + builtin_file = files.get_abs_path(f"api/{path}.py") + if files.is_in_dir(builtin_file, files.get_abs_path("api")) and files.exists( + builtin_file + ): + classes = load_classes_from_file(builtin_file, ApiHandler) + if classes: + handler_cls = classes[0] + + # Check plugin api folders: path format plugins// + if handler_cls is None and path.startswith("plugins/"): + parts = path.split("/", 2) + if len(parts) == 3: + _, plugin_name, handler_name = parts + plugin_dir = plugins.find_plugin_dir(plugin_name) + if plugin_dir: + plugin_file = Path(plugin_dir) / "api" / f"{handler_name}.py" + if plugin_file.is_file(): + classes = load_classes_from_file(str(plugin_file), ApiHandler) + if classes: + handler_cls = classes[0] + + # Check user api/.py + if handler_cls is None: + user_api_dir = files.get_abs_path(files.USER_DIR, files.API_DIR) + user_file = files.get_abs_path(user_api_dir, f"{path}.py") + if files.is_in_dir(user_file, user_api_dir) and files.exists(user_file): + classes = load_classes_from_file(user_file, ApiHandler) + if classes: + handler_cls = classes[0] + + if handler_cls is None: + return Response(f"API endpoint not found: {path}", 404) + + # Check method is allowed + if request.method not in handler_cls.get_methods(): + return Response(f"Method {request.method} not allowed for: {path}", 405) + + # Build handler call, wrapping with security decorators as required + async def call_handler() -> BaseResponse: + instance = handler_cls(app, lock) + return await instance.handle_request(request=request) + + handler_fn = call_handler + if handler_cls.requires_csrf(): + handler_fn = csrf_protect(handler_fn) + if handler_cls.requires_api_key(): + handler_fn = requires_api_key(handler_fn) + if handler_cls.requires_auth(): + handler_fn = requires_auth(handler_fn) + if handler_cls.requires_loopback(): + handler_fn = requires_loopback(handler_fn) + + cache.add(CACHE_AREA, path, handler_fn) + return await handler_fn() + + app.add_url_rule( + "/api/", + "api_dispatch", + _dispatch, + methods=["GET", "POST", "PUT", "PATCH", "DELETE"], + ) + + +def register_watchdogs(): + from helpers import watchdog + from helpers.ws import CACHE_AREA as WS_CACHE_AREA + + + def on_api_change(items: list[watchdog.WatchItem]): + PrintStyle.debug("API endpoint watchdog triggered:", items) + cache.clear(CACHE_AREA) + cache.clear(WS_CACHE_AREA) + + watchdog.add_watchdog( + "api_handlers", + roots=[ + files.get_abs_path(files.API_DIR), + files.get_abs_path(files.USER_DIR, files.API_DIR), + ], + patterns=["*.py"], + handler=on_api_change, + ) diff --git a/helpers/api.py.dox.md b/helpers/api.py.dox.md index 373a53ab2ad8d9ec27e8d458f86ffabc61b8bd47..5de8d844c95871f6f94bdd882a715c00f80e9406 100644 --- a/helpers/api.py.dox.md +++ b/helpers/api.py.dox.md @@ -1,3 +1,72 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:167de11807ea17617a462adcd716f5e56393617d090a3c9c77eead2216f8f19d -size 3839 +# api.py DOX + +## Purpose + +- Own the `api.py` helper module. +- This module defines API handler registration, request security gates, CSRF checks, and watchdog registration. +- Keep this file-level DOX profile synchronized with `api.py` because this directory is intentionally flat. + +## Ownership + +- `api.py` owns the runtime implementation. +- `api.py.dox.md` owns durable notes about responsibilities, contracts, side effects, and verification for that implementation. +- Classes: +- `ApiHandler` (no explicit base class) + - `requires_loopback(cls) -> bool` + - `requires_api_key(cls) -> bool` + - `requires_auth(cls) -> bool` + - `get_methods(cls) -> list[str]` + - `requires_csrf(cls) -> bool` + - `async process(self, input: Input, request: Request) -> Output` + - `async handle_request(self, request: Request) -> Response` + - `use_context(self, ctxid: str, create_if_not_exists: bool=...)` +- Top-level functions: +- `requires_api_key(f)` +- `requires_loopback(f)` +- `requires_auth(f)` +- `csrf_protect(f)` +- `register_api_route(app: Flask, lock: ThreadLockType) -> None` +- `register_watchdogs()` +- Notable constants/configuration names: `CACHE_AREA`. + +## Runtime Contracts + +- Helper modules own reusable framework APIs and must preserve public callers unless all callers, tests, and docs are updated together. +- Update this file whenever public functions, classes, persistence behavior, path/security assumptions, side effects, or cross-module contracts change. +- `ApiHandler` defines `process(...)`. +- `ApiHandler` defines `get_methods(...)`. +- `ApiHandler` defines `requires_auth(...)`. +- `ApiHandler` defines `requires_csrf(...)`. +- `ApiHandler` defines `requires_api_key(...)`. +- `ApiHandler` defines `requires_loopback(...)`. +- Observed side-effect areas: filesystem reads, filesystem writes, filesystem deletion, WebSocket state, plugin state, settings/state persistence, secret handling. +- Imported dependency areas include: `abc`, `flask`, `functools`, `helpers`, `helpers.errors`, `helpers.network`, `helpers.print_style`, `json`, `pathlib`, `threading`, `typing`, `werkzeug.wrappers.response`. + +## Key Concepts + +- Important called helpers/classes observed in the source: `wraps`, `app.add_url_rule`, `watchdog.add_watchdog`, `cls.requires_auth`, `_use_context`, `login.get_credentials_hash`, `files.get_abs_path`, `handler_cls.requires_csrf`, `handler_cls.requires_api_key`, `handler_cls.requires_auth`, `handler_cls.requires_loopback`, `cache.add`, `PrintStyle.debug`, `cache.clear`, `get_settings`, `f`, `is_loopback_address`, `Response`, `redirect`, `files.is_in_dir`. +- HTTP handlers retain built-in `api/` and explicit plugin API precedence, then fall back to standalone `usr/api/`; built-in and user roots are containment-checked, and every loaded handler keeps its declared authentication, CSRF, API-key, loopback, and method gates. +- Keep request/response, tool, or helper semantics documented here at the same time as source changes. + +## Work Guidance + +- Preserve public helper APIs used by core code and plugins unless every caller is updated. +- Keep path, auth, secret, persistence, network, and subprocess behavior explicit and bounded. +- Prefer adding cohesive helper functions here only when behavior is reused across modules. + +## Verification + +- Run targeted tests for changed helper behavior; run security regressions for auth, filesystem, WebSocket, tunnel, upload, or secret-handling helpers. +- Related tests observed by source search: + - `tests/test_api_chat_lifetime.py` + - `tests/test_browser_agent_regressions.py` + - `tests/test_download_toast_regressions.py` + - `tests/test_fasta2a_client.py` + - `tests/test_fastmcp_openapi_security.py` + - `tests/test_host_browser_connector.py` + - `tests/test_image_get_security.py` + - `tests/test_model_config_api_keys.py` + +## Child DOX Index + +No child DOX files. diff --git a/helpers/attachment_manager.py b/helpers/attachment_manager.py index bad83c49716732280475a9805b0b31fa29903b76..cf366dadfa109d93cfae9c8c4c72cb1f96130e34 100644 --- a/helpers/attachment_manager.py +++ b/helpers/attachment_manager.py @@ -1,3 +1,94 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:b453b1d6f4241c2a22790529624c5127377b875e0f22ed65cbbe59907ee0e8a9 -size 3138 +import os +import io +import base64 +from PIL import Image +from typing import Dict, List, Optional, Tuple +from helpers.security import safe_filename +from werkzeug.datastructures import FileStorage + +from helpers.print_style import PrintStyle + +class AttachmentManager: + ALLOWED_EXTENSIONS = { + 'image': {'jpg', 'jpeg', 'png', 'bmp'}, + 'code': {'py', 'js', 'sh', 'html', 'css'}, + 'document': {'md', 'pdf', 'txt', 'csv', 'json'} + } + + def __init__(self, work_dir: str): + self.work_dir = work_dir + os.makedirs(work_dir, exist_ok=True) + + def is_allowed_file(self, filename: str) -> bool: + ext = self.get_file_extension(filename) + all_allowed = set().union(*self.ALLOWED_EXTENSIONS.values()) + return ext in all_allowed + + def get_file_type(self, filename: str) -> str: + ext = self.get_file_extension(filename) + for file_type, extensions in self.ALLOWED_EXTENSIONS.items(): + if ext in extensions: + return file_type + return 'unknown' + + @staticmethod + def get_file_extension(filename: str) -> str: + return filename.rsplit('.', 1)[1].lower() if '.' in filename else '' + + def validate_mime_type(self, file) -> bool: + try: + mime_type = file.content_type + return mime_type.split('/')[0] in ['image', 'text', 'application'] + except AttributeError: + return False + + def save_file(self, file: FileStorage, name: str) -> Tuple[str, Dict]: + """Save file and return path and metadata""" + try: + filename = safe_filename(name) + if not filename: + raise ValueError("Invalid filename") + + file_path = os.path.join(self.work_dir, filename) + + file_type = self.get_file_type(filename) + metadata = { + 'filename': filename, + 'type': file_type, + 'extension': self.get_file_extension(filename), + 'preview': None + } + + # Save file + file.save(file_path) + + # Generate preview for images + if file_type == 'image': + metadata['preview'] = self.generate_image_preview(file_path) + + return file_path, metadata + + except Exception as e: + PrintStyle.error(f"Error saving file {name}: {e}") + return None, {} # type: ignore + + def generate_image_preview(self, image_path: str, max_size: int = 800) -> Optional[str]: + try: + with Image.open(image_path) as img: + # Convert image if needed + if img.mode in ('RGBA', 'P'): + img = img.convert('RGB') + + # Resize for preview + img.thumbnail((max_size, max_size)) + + # Save to buffer + buffer = io.BytesIO() + img.save(buffer, format="JPEG", quality=70, optimize=True) + + # Convert to base64 + return base64.b64encode(buffer.getvalue()).decode('utf-8') + except Exception as e: + PrintStyle.error(f"Error generating preview for {image_path}: {e}") + return None + \ No newline at end of file diff --git a/helpers/attachment_manager.py.dox.md b/helpers/attachment_manager.py.dox.md index 3e942b51bc742eaad0f838f7767d5499d3c96669..46dd4a93022cfd0119433ba46d6df0b876a0d344 100644 --- a/helpers/attachment_manager.py.dox.md +++ b/helpers/attachment_manager.py.dox.md @@ -1,3 +1,47 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:9e9e5fbada8cf044337a0690dd7b1688e195a870f589b0ed214992743b7bc1a3 -size 2663 +# attachment_manager.py DOX + +## Purpose + +- Own the `attachment_manager.py` helper module. +- This module tracks uploaded or generated attachments associated with chat contexts. +- Keep this file-level DOX profile synchronized with `attachment_manager.py` because this directory is intentionally flat. + +## Ownership + +- `attachment_manager.py` owns the runtime implementation. +- `attachment_manager.py.dox.md` owns durable notes about responsibilities, contracts, side effects, and verification for that implementation. +- Classes: +- `AttachmentManager` (no explicit base class) + - `is_allowed_file(self, filename: str) -> bool` + - `get_file_type(self, filename: str) -> str` + - `get_file_extension(filename: str) -> str` + - `validate_mime_type(self, file) -> bool` + - `save_file(self, file: FileStorage, name: str) -> Tuple[str, Dict]` + - `generate_image_preview(self, image_path: str, max_size: int=...) -> Optional[str]` + +## Runtime Contracts + +- Helper modules own reusable framework APIs and must preserve public callers unless all callers, tests, and docs are updated together. +- Update this file whenever public functions, classes, persistence behavior, path/security assumptions, side effects, or cross-module contracts change. +- Observed side-effect areas: filesystem reads, filesystem writes, settings/state persistence. +- Imported dependency areas include: `PIL`, `base64`, `helpers.print_style`, `helpers.security`, `io`, `os`, `typing`, `werkzeug.datastructures`. + +## Key Concepts + +- Important called helpers/classes observed in the source: `os.makedirs`, `self.get_file_extension`, `set.union`, `filename.rsplit.lower`, `safe_filename`, `os.path.join`, `self.get_file_type`, `file.save`, `ValueError`, `self.generate_image_preview`, `PrintStyle.error`, `img.thumbnail`, `io.BytesIO`, `img.save`, `base64.b64encode.decode`, `mime_type.split`, `img.convert`, `filename.rsplit`, `base64.b64encode`, `buffer.getvalue`. +- Keep request/response, tool, or helper semantics documented here at the same time as source changes. + +## Work Guidance + +- Preserve public helper APIs used by core code and plugins unless every caller is updated. +- Keep path, auth, secret, persistence, network, and subprocess behavior explicit and bounded. +- Prefer adding cohesive helper functions here only when behavior is reused across modules. + +## Verification + +- Run targeted tests for changed helper behavior; run security regressions for auth, filesystem, WebSocket, tunnel, upload, or secret-handling helpers. +- No direct test reference was found by name search; choose the nearest behavioral test or perform a focused smoke check. + +## Child DOX Index + +No child DOX files. diff --git a/helpers/backup.py b/helpers/backup.py index f248ca4e6e16d835459e1ed6ef78886c9371eabe..f97ff67af5ab7fdf4733feea2e551bc3d0bbd9da 100644 --- a/helpers/backup.py +++ b/helpers/backup.py @@ -1,3 +1,859 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:13ec621ba14566bbdcfdaf93f4f88927b706ff159aebe5bb9e39ec7ee387aa90 -size 38465 +import zipfile +import json +import os +import tempfile +import datetime +import platform +from typing import List, Dict, Any, Optional + +from pathspec import PathSpec + +from helpers import files, runtime, git, dotenv +from helpers.localization import Localization +from helpers.print_style import PrintStyle + + +class BackupService: + """ + Core backup and restore service for Agent Zero. + + Features: + - JSON-based metadata with user-editable path specifications + - Comprehensive system information collection + - Checksum validation for integrity + - RFC compatibility through existing file helpers + - Git version integration consistent with main application + """ + + def __init__(self): + self.agent_zero_version = self._get_agent_zero_version() + self.agent_zero_root = files.get_abs_path("") # Resolved Agent Zero root + + # Build base paths map for pattern resolution + self.base_paths = { + self.agent_zero_root: self.agent_zero_root, + } + + def get_default_backup_metadata(self) -> Dict[str, Any]: + """Get default backup patterns and metadata""" + timestamp = Localization.get().now_iso() + + default_patterns = self._get_default_patterns() + include_patterns, exclude_patterns = self._parse_patterns(default_patterns) + + return { + "backup_name": f"agent-zero-backup-{timestamp[:10]}", + "include_hidden": True, + "include_patterns": include_patterns, + "exclude_patterns": exclude_patterns, + "backup_config": { + "compression_level": 6, + "integrity_check": True + } + } + + def _get_default_patterns(self) -> str: + """Get default backup patterns with resolved absolute paths. + + Only includes Agent Zero project directory patterns. + """ + # Ensure paths don't have double slashes + agent_root = self.agent_zero_root.rstrip('/') + + return f"""# User data +# All persistent user data is now centralized in /usr for easier backup and restore +{agent_root}/usr/** +!{agent_root}/usr/.time_travel/** +""" + + def _get_agent_zero_version(self) -> str: + """Get current Agent Zero version""" + try: + # Get version from git info (same as run_ui.py) + gitinfo = git.get_git_info() + return gitinfo.get("version", "development") + except Exception: + return "unknown" + + def _resolve_path(self, pattern_path: str) -> str: + """Resolve pattern path to absolute system path (now patterns are already absolute)""" + return pattern_path + + def _unresolve_path(self, abs_path: str) -> str: + """Convert absolute path back to pattern path (now patterns are already absolute)""" + return abs_path + + def _parse_patterns(self, patterns: str) -> tuple[list[str], list[str]]: + """Parse patterns string into include and exclude pattern arrays""" + include_patterns = [] + exclude_patterns = [] + + for line in patterns.split('\n'): + line = line.strip() + if not line or line.startswith('#'): + continue + + if line.startswith('!'): + # Exclude pattern + exclude_patterns.append(line[1:]) # Remove the '!' prefix + else: + # Include pattern + include_patterns.append(line) + + return include_patterns, exclude_patterns + + def _patterns_to_string(self, include_patterns: list[str], exclude_patterns: list[str]) -> str: + """Convert pattern arrays back to patterns string for pathspec processing""" + patterns = [] + + # Add include patterns + for pattern in include_patterns: + patterns.append(pattern) + + # Add exclude patterns with '!' prefix + for pattern in exclude_patterns: + patterns.append(f"!{pattern}") + + return '\n'.join(patterns) + + async def _get_system_info(self) -> Dict[str, Any]: + """Collect system information for metadata""" + import psutil + + try: + return { + "platform": platform.platform(), + "system": platform.system(), + "release": platform.release(), + "version": platform.version(), + "machine": platform.machine(), + "processor": platform.processor(), + "architecture": platform.architecture()[0], + "hostname": platform.node(), + "python_version": platform.python_version(), + "cpu_count": str(psutil.cpu_count()), + "memory_total": str(psutil.virtual_memory().total), + "disk_usage": str(psutil.disk_usage('/').total if os.path.exists('/') else 0) + } + except Exception as e: + return {"error": f"Failed to collect system info: {str(e)}"} + + async def _get_environment_info(self) -> Dict[str, Any]: + """Collect environment information for metadata""" + try: + return { + "user": os.environ.get("USER", "unknown"), + "home": os.environ.get("HOME", "unknown"), + "shell": os.environ.get("SHELL", "unknown"), + "path": os.environ.get("PATH", "")[:200] + "..." if len(os.environ.get("PATH", "")) > 200 else os.environ.get("PATH", ""), + "timezone": Localization.get().get_timezone(), + "working_directory": os.getcwd(), + "agent_zero_root": files.get_abs_path(""), + "runtime_mode": "development" if runtime.is_development() else "production" + } + except Exception as e: + return {"error": f"Failed to collect environment info: {str(e)}"} + + async def _get_backup_author(self) -> str: + """Get backup author/system identifier""" + try: + import getpass + username = getpass.getuser() + hostname = platform.node() + return f"{username}@{hostname}" + except Exception: + return "unknown" + + def _count_directories(self, matched_files: List[Dict[str, Any]]) -> int: + """Count unique directories in file list""" + directories = set() + for file_info in matched_files: + dir_path = os.path.dirname(file_info["path"]) + if dir_path: + directories.add(dir_path) + return len(directories) + + def _get_explicit_patterns(self, include_patterns: List[str]) -> set[str]: + """Extract explicit (non-wildcard) patterns that should always be included""" + explicit_patterns = set() + + for pattern in include_patterns: + # If pattern doesn't contain wildcards, it's explicit + if '*' not in pattern and '?' not in pattern: + # Remove leading slash for comparison + explicit_patterns.add(pattern.lstrip('/')) + + # Also add parent directories as explicit (so hidden dirs can be traversed) + path_parts = pattern.lstrip('/').split('/') + for i in range(1, len(path_parts)): + parent_path = '/'.join(path_parts[:i]) + explicit_patterns.add(parent_path) + + return explicit_patterns + + def _is_explicitly_included(self, file_path: str, explicit_patterns: set[str]) -> bool: + """Check if a file/directory is explicitly included in patterns""" + relative_path = file_path.lstrip('/') + return relative_path in explicit_patterns + + def _translate_patterns(self, patterns: List[str], backup_metadata: Dict[str, Any]) -> List[str]: + """Translate patterns from backed up system to current system. + + Replaces the backed up Agent Zero root path with the current Agent Zero root path + in all patterns if there's an exact match at the beginning of the pattern. + + Args: + patterns: List of patterns from the backed up system + backup_metadata: Backup metadata containing the original agent_zero_root + + Returns: + List of translated patterns for the current system + """ + # Get the backed up agent zero root path from metadata + environment_info = backup_metadata.get("environment_info", {}) + backed_up_agent_root = environment_info.get("agent_zero_root", "") + + # Get current agent zero root path + current_agent_root = self.agent_zero_root + + # If we don't have the backed up root path, return patterns as-is + if not backed_up_agent_root: + return patterns + + # Ensure paths have consistent trailing slash handling + backed_up_agent_root = backed_up_agent_root.rstrip('/') + current_agent_root = current_agent_root.rstrip('/') + + translated_patterns = [] + for pattern in patterns: + # Check if the pattern starts with the backed up agent zero root + if pattern.startswith(backed_up_agent_root + '/') or pattern == backed_up_agent_root: + # Replace the backed up root with the current root + relative_pattern = pattern[len(backed_up_agent_root):].lstrip('/') + if relative_pattern: + translated_pattern = current_agent_root + '/' + relative_pattern + else: + translated_pattern = current_agent_root + translated_patterns.append(translated_pattern) + else: + # Pattern doesn't start with backed up agent root, keep as-is + translated_patterns.append(pattern) + + return translated_patterns + + async def test_patterns(self, metadata: Dict[str, Any], max_files: Optional[int] = 1000) -> List[Dict[str, Any]]: + """Test backup patterns and return list of matched files. + + Pass max_files=None for internal flows that must process the complete + match set, such as backup creation and restore cleanup. + """ + include_patterns = metadata.get("include_patterns", []) + exclude_patterns = metadata.get("exclude_patterns", []) + include_hidden = metadata.get("include_hidden", True) + + # Convert to patterns string for pathspec + patterns_string = self._patterns_to_string(include_patterns, exclude_patterns) + + # Parse patterns using pathspec + pattern_lines = [line.strip() for line in patterns_string.split('\n') if line.strip() and not line.strip().startswith('#')] + + if not pattern_lines: + return [] + + # Get explicit patterns for hidden file handling + explicit_patterns = self._get_explicit_patterns(include_patterns) + + has_limit = max_files is not None + matched_files = [] + processed_count = 0 + + try: + spec = PathSpec.from_lines("gitignore", pattern_lines) + + # Walk through base directories + for base_pattern_path, base_real_path in self.base_paths.items(): + if not os.path.exists(base_real_path): + continue + + for root, dirs, files_list in os.walk(base_real_path): + # Filter hidden directories if not included, BUT allow explicit ones + if not include_hidden: + dirs_to_keep = [] + for d in dirs: + if not d.startswith('.'): + dirs_to_keep.append(d) + else: + # Check if this hidden directory is explicitly included + dir_path = os.path.join(root, d) + pattern_path = self._unresolve_path(dir_path) + if self._is_explicitly_included(pattern_path, explicit_patterns): + dirs_to_keep.append(d) + dirs[:] = dirs_to_keep + + for file in files_list: + if has_limit and processed_count >= max_files: + break + + file_path = os.path.join(root, file) + pattern_path = self._unresolve_path(file_path) + + # Skip hidden files if not included, BUT allow explicit ones + if not include_hidden and file.startswith('.'): + if not self._is_explicitly_included(pattern_path, explicit_patterns): + continue + + # Remove leading slash for pathspec matching + relative_path = pattern_path.lstrip('/') + + if spec.match_file(relative_path): + try: + stat = os.stat(file_path) + matched_files.append({ + "path": pattern_path, + "real_path": file_path, + "size": stat.st_size, + "modified": datetime.datetime.fromtimestamp( + stat.st_mtime, + tz=Localization.get().get_tzinfo(), + ).isoformat(), + "type": "file" + }) + processed_count += 1 + except (OSError, IOError): + # Skip files we can't access + continue + + if has_limit and processed_count >= max_files: + break + + if has_limit and processed_count >= max_files: + break + + except Exception as e: + raise Exception(f"Error processing patterns: {str(e)}") + + return matched_files + + async def create_backup( + self, + include_patterns: List[str], + exclude_patterns: List[str], + include_hidden: bool = True, + backup_name: str = "agent-zero-backup" + ) -> str: + """Create backup archive and return path to created file""" + + # Create metadata for test_patterns + metadata = { + "include_patterns": include_patterns, + "exclude_patterns": exclude_patterns, + "include_hidden": include_hidden + } + + # Get the complete matched file set. Preview and dry-run callers may + # cap their scans for UI responsiveness, but the archive itself must be + # complete. + matched_files = await self.test_patterns(metadata, max_files=None) + + if not matched_files: + raise Exception("No files matched the backup patterns") + + # Create temporary zip file + temp_dir = tempfile.mkdtemp() + zip_path = os.path.join(temp_dir, f"{backup_name}.zip") + + try: + with zipfile.ZipFile(zip_path, 'w', zipfile.ZIP_DEFLATED) as zipf: + # Add comprehensive metadata + metadata = { + # Basic backup information + "agent_zero_version": self.agent_zero_version, + "timestamp": Localization.get().now_iso(), + "backup_name": backup_name, + "include_hidden": include_hidden, + + # Pattern arrays for granular control during restore + "include_patterns": include_patterns, + "exclude_patterns": exclude_patterns, + + # System and environment information + "system_info": await self._get_system_info(), + "environment_info": await self._get_environment_info(), + "backup_author": await self._get_backup_author(), + + # Backup configuration + "backup_config": { + "include_patterns": include_patterns, + "exclude_patterns": exclude_patterns, + "include_hidden": include_hidden, + "compression_level": 6, + "integrity_check": True + }, + + # File information + "files": [ + { + "path": f["path"], + "size": f["size"], + "modified": f["modified"], + "type": "file" + } + for f in matched_files + ], + + # Statistics + "total_files": len(matched_files), + "backup_size": sum(f["size"] for f in matched_files), + "directory_count": self._count_directories(matched_files), + } + + zipf.writestr("metadata.json", json.dumps(metadata, indent=2)) + + # Add files + for file_info in matched_files: + real_path = file_info["real_path"] + archive_path = file_info["path"].lstrip('/') + + try: + if os.path.exists(real_path) and os.path.isfile(real_path): + zipf.write(real_path, archive_path) + except (OSError, IOError) as e: + # Log error but continue with other files + PrintStyle().warning(f"Warning: Could not backup file {real_path}: {e}") + continue + + return zip_path + + except Exception as e: + # Cleanup on error + if os.path.exists(zip_path): + os.remove(zip_path) + raise Exception(f"Error creating backup: {str(e)}") + + async def inspect_backup(self, backup_file) -> Dict[str, Any]: + """Inspect backup archive and return metadata""" + + # Save uploaded file temporarily + temp_dir = tempfile.mkdtemp() + temp_file = os.path.join(temp_dir, "backup.zip") + + try: + backup_file.save(temp_file) + + with zipfile.ZipFile(temp_file, 'r') as zipf: + # Read metadata + if "metadata.json" not in zipf.namelist(): + raise Exception("Invalid backup file: missing metadata.json") + + metadata_content = zipf.read("metadata.json").decode('utf-8') + metadata = json.loads(metadata_content) + + # Add file list from archive + files_in_archive = [name for name in zipf.namelist() if name != "metadata.json"] + metadata["files_in_archive"] = files_in_archive + + return metadata + + except zipfile.BadZipFile: + raise Exception("Invalid backup file: not a valid zip archive") + except json.JSONDecodeError: + raise Exception("Invalid backup file: corrupted metadata") + finally: + # Cleanup + if os.path.exists(temp_file): + os.remove(temp_file) + if os.path.exists(temp_dir): + os.rmdir(temp_dir) + + async def preview_restore( + self, + backup_file, + restore_include_patterns: Optional[List[str]] = None, + restore_exclude_patterns: Optional[List[str]] = None, + overwrite_policy: str = "overwrite", + clean_before_restore: bool = False, + user_edited_metadata: Optional[Dict[str, Any]] = None + ) -> Dict[str, Any]: + """Preview which files would be restored based on patterns""" + + # Save uploaded file temporarily + temp_dir = tempfile.mkdtemp() + temp_file = os.path.join(temp_dir, "backup.zip") + + files_to_restore = [] + skipped_files = [] + + try: + backup_file.save(temp_file) + + with zipfile.ZipFile(temp_file, 'r') as zipf: + # Read backup metadata from archive + original_backup_metadata = {} + if "metadata.json" in zipf.namelist(): + metadata_content = zipf.read("metadata.json").decode('utf-8') + original_backup_metadata = json.loads(metadata_content) + + # Use user-edited metadata if provided, otherwise fall back to original + backup_metadata = user_edited_metadata if user_edited_metadata else original_backup_metadata + + # Get files from archive (excluding metadata files) + archive_files = [name for name in zipf.namelist() + if name not in ["metadata.json", "checksums.json"]] + + # Create pathspec for restore patterns if provided + restore_spec = None + if restore_include_patterns or restore_exclude_patterns: + pattern_lines = [] + if restore_include_patterns: + # Translate patterns from backed up system to current system + translated_include_patterns = self._translate_patterns(restore_include_patterns, original_backup_metadata) + for pattern in translated_include_patterns: + # Remove leading slash for pathspec matching + pattern_lines.append(pattern.lstrip('/')) + if restore_exclude_patterns: + # Translate patterns from backed up system to current system + translated_exclude_patterns = self._translate_patterns(restore_exclude_patterns, original_backup_metadata) + for pattern in translated_exclude_patterns: + # Remove leading slash for pathspec matching + pattern_lines.append(f"!{pattern.lstrip('/')}") + + if pattern_lines: + from pathspec import PathSpec + restore_spec = PathSpec.from_lines("gitignore", pattern_lines) + + # Process each file in archive + for archive_path in archive_files: + # Archive path is already the correct relative path (e.g., "a0/tmp/settings.json") + original_path = archive_path + + # Translate path from backed up system to current system + # Use original metadata for path translation (environment_info needed for this) + target_path = self._translate_restore_path(archive_path, original_backup_metadata) + + # For pattern matching, we need to use the translated path (current system) + # so that patterns like "/home/rafael/a0/data/**" can match files correctly + translated_path_for_matching = target_path.lstrip('/') + + # Check if file matches restore patterns + if restore_spec and not restore_spec.match_file(translated_path_for_matching): + skipped_files.append({ + "archive_path": archive_path, + "original_path": original_path, + "reason": "not_matched_by_pattern" + }) + continue + + # Check file conflict policy for existing files + if os.path.exists(target_path): + if overwrite_policy == "skip": + skipped_files.append({ + "archive_path": archive_path, + "original_path": original_path, + "reason": "file_exists_skip_policy" + }) + continue + + # File will be restored + files_to_restore.append({ + "archive_path": archive_path, + "original_path": original_path, + "target_path": target_path, + "action": "restore" + }) + + # Handle clean before restore if requested + files_to_delete = [] + if clean_before_restore: + # Use user-edited metadata for clean operations so patterns from ACE editor are used + files_to_delete = await self._find_files_to_clean_with_user_metadata(backup_metadata, original_backup_metadata) + + # Combine delete and restore operations for preview + all_operations = files_to_delete + files_to_restore + + return { + "files": all_operations, + "files_to_delete": files_to_delete, + "files_to_restore": files_to_restore, + "skipped_files": skipped_files, + "total_count": len(all_operations), + "delete_count": len(files_to_delete), + "restore_count": len(files_to_restore), + "skipped_count": len(skipped_files), + "backup_metadata": backup_metadata, # Return user-edited metadata + "overwrite_policy": overwrite_policy, + "clean_before_restore": clean_before_restore + } + + except zipfile.BadZipFile: + raise Exception("Invalid backup file: not a valid zip archive") + except json.JSONDecodeError: + raise Exception("Invalid backup file: corrupted metadata") + except Exception as e: + raise Exception(f"Error previewing restore: {str(e)}") + finally: + # Cleanup + if os.path.exists(temp_file): + os.remove(temp_file) + if os.path.exists(temp_dir): + os.rmdir(temp_dir) + + async def restore_backup( + self, + backup_file, + restore_include_patterns: Optional[List[str]] = None, + restore_exclude_patterns: Optional[List[str]] = None, + overwrite_policy: str = "overwrite", + clean_before_restore: bool = False, + user_edited_metadata: Optional[Dict[str, Any]] = None + ) -> Dict[str, Any]: + """Restore files from backup archive""" + + allowed_origins = dotenv.get_dotenv_value("ALLOWED_ORIGINS", "") + dotenv_path = os.path.abspath(dotenv.get_dotenv_file_path()) + + # Save uploaded file temporarily + temp_dir = tempfile.mkdtemp() + temp_file = os.path.join(temp_dir, "backup.zip") + + restored_files = [] + skipped_files = [] + errors = [] + deleted_files = [] + + try: + backup_file.save(temp_file) + + with zipfile.ZipFile(temp_file, 'r') as zipf: + # Read backup metadata from archive + original_backup_metadata = {} + if "metadata.json" in zipf.namelist(): + metadata_content = zipf.read("metadata.json").decode('utf-8') + original_backup_metadata = json.loads(metadata_content) + + # Use user-edited metadata if provided, otherwise fall back to original + backup_metadata = user_edited_metadata if user_edited_metadata else original_backup_metadata + + # Perform clean before restore if requested + if clean_before_restore: + # Use user-edited metadata for clean operations so patterns from ACE editor are used + files_to_delete = await self._find_files_to_clean_with_user_metadata(backup_metadata, original_backup_metadata) + for delete_info in files_to_delete: + try: + real_path = delete_info["real_path"] + if os.path.exists(real_path) and os.path.isfile(real_path): + os.remove(real_path) + deleted_files.append({ + "path": delete_info["path"], + "real_path": real_path, + "action": "deleted", + "reason": "clean_before_restore" + }) + except Exception as e: + errors.append({ + "path": delete_info["path"], + "real_path": delete_info.get("real_path", "unknown"), + "error": f"Failed to delete: {str(e)}" + }) + + # Get files from archive (excluding metadata files) + archive_files = [name for name in zipf.namelist() + if name not in ["metadata.json", "checksums.json"]] + + # Create pathspec for restore patterns if provided + restore_spec = None + if restore_include_patterns or restore_exclude_patterns: + pattern_lines = [] + if restore_include_patterns: + # Translate patterns from backed up system to current system + translated_include_patterns = self._translate_patterns(restore_include_patterns, original_backup_metadata) + for pattern in translated_include_patterns: + # Remove leading slash for pathspec matching + pattern_lines.append(pattern.lstrip('/')) + if restore_exclude_patterns: + # Translate patterns from backed up system to current system + translated_exclude_patterns = self._translate_patterns(restore_exclude_patterns, original_backup_metadata) + for pattern in translated_exclude_patterns: + # Remove leading slash for pathspec matching + pattern_lines.append(f"!{pattern.lstrip('/')}") + + if pattern_lines: + from pathspec import PathSpec + restore_spec = PathSpec.from_lines("gitignore", pattern_lines) + + # Process each file in archive + for archive_path in archive_files: + # Archive path is already the correct relative path (e.g., "a0/tmp/settings.json") + original_path = archive_path + + # Translate path from backed up system to current system + # Use original metadata for path translation (environment_info needed for this) + target_path = self._translate_restore_path(archive_path, original_backup_metadata) + + # For pattern matching, we need to use the translated path (current system) + # so that patterns like "/home/rafael/a0/data/**" can match files correctly + translated_path_for_matching = target_path.lstrip('/') + + # Check if file matches restore patterns + if restore_spec and not restore_spec.match_file(translated_path_for_matching): + skipped_files.append({ + "archive_path": archive_path, + "original_path": original_path, + "reason": "not_matched_by_pattern" + }) + continue + + try: + # Handle overwrite policy + if os.path.exists(target_path): + if overwrite_policy == "skip": + skipped_files.append({ + "archive_path": archive_path, + "original_path": original_path, + "reason": "file_exists_skip_policy" + }) + continue + elif overwrite_policy == "backup": + timestamp = Localization.get().now().strftime('%Y%m%d_%H%M%S') + backup_path = f"{target_path}.backup.{timestamp}" + import shutil + shutil.move(target_path, backup_path) + + # Create target directory if needed + target_dir = os.path.dirname(target_path) + if target_dir: + os.makedirs(target_dir, exist_ok=True) + + # Extract file + import shutil + with zipf.open(archive_path) as source, open(target_path, 'wb') as target: + shutil.copyfileobj(source, target) + + if os.path.abspath(target_path) == dotenv_path: + dotenv.save_dotenv_value( + "ALLOWED_ORIGINS", allowed_origins, reload_env=False + ) + + restored_files.append({ + "archive_path": archive_path, + "original_path": original_path, + "target_path": target_path, + "status": "restored" + }) + + except Exception as e: + errors.append({ + "path": archive_path, + "original_path": original_path, + "error": str(e) + }) + + return { + "restored_files": restored_files, + "deleted_files": deleted_files, + "skipped_files": skipped_files, + "errors": errors, + "backup_metadata": backup_metadata, # Return user-edited metadata + "clean_before_restore": clean_before_restore + } + + except zipfile.BadZipFile: + raise Exception("Invalid backup file: not a valid zip archive") + except json.JSONDecodeError: + raise Exception("Invalid backup file: corrupted metadata") + except Exception as e: + raise Exception(f"Error restoring backup: {str(e)}") + finally: + # Cleanup + if os.path.exists(temp_file): + os.remove(temp_file) + if os.path.exists(temp_dir): + os.rmdir(temp_dir) + + def _translate_restore_path(self, archive_path: str, backup_metadata: Dict[str, Any]) -> str: + """Translate file path from backed up system to current system. + + Replaces the backed up Agent Zero root path with the current Agent Zero root path + if there's an exact match at the beginning of the path. + + Args: + archive_path: Original file path from the archive + backup_metadata: Backup metadata containing the original agent_zero_root + + Returns: + Translated path for the current system + """ + # Get the backed up agent zero root path from metadata + environment_info = backup_metadata.get("environment_info", {}) + backed_up_agent_root = environment_info.get("agent_zero_root", "") + + # Get current agent zero root path + current_agent_root = self.agent_zero_root + + # If we don't have the backed up root path, use original path with leading slash + if not backed_up_agent_root: + return "/" + archive_path.lstrip('/') + + # Ensure paths have consistent trailing slash handling + backed_up_agent_root = backed_up_agent_root.rstrip('/') + current_agent_root = current_agent_root.rstrip('/') + + # Convert archive path to absolute path (add leading slash if missing) + if not archive_path.startswith('/'): + absolute_archive_path = "/" + archive_path + else: + absolute_archive_path = archive_path + + # Check if the archive path starts with the backed up agent zero root + if absolute_archive_path.startswith(backed_up_agent_root + '/') or absolute_archive_path == backed_up_agent_root: + # Replace the backed up root with the current root + relative_path = absolute_archive_path[len(backed_up_agent_root):].lstrip('/') + if relative_path: + translated_path = current_agent_root + '/' + relative_path + else: + translated_path = current_agent_root + return translated_path + else: + # Path doesn't start with backed up agent root, return as-is + return absolute_archive_path + + async def _find_files_to_clean_with_user_metadata(self, user_metadata: Dict[str, Any], original_metadata: Dict[str, Any]) -> List[Dict[str, Any]]: + """Find existing files that match patterns from user-edited metadata for clean operations""" + # Use user-edited patterns for what to clean + user_include_patterns = user_metadata.get("include_patterns", []) + user_exclude_patterns = user_metadata.get("exclude_patterns", []) + include_hidden = user_metadata.get("include_hidden", True) + + if not user_include_patterns: + return [] + + # Translate user-edited patterns from backed up system to current system + # Use original metadata for path translation (environment_info) + translated_include_patterns = self._translate_patterns(user_include_patterns, original_metadata) + translated_exclude_patterns = self._translate_patterns(user_exclude_patterns, original_metadata) + + # Create metadata object for testing translated patterns + metadata = { + "include_patterns": translated_include_patterns, + "exclude_patterns": translated_exclude_patterns, + "include_hidden": include_hidden + } + + # Find existing files that match the translated user-edited patterns + try: + existing_files = await self.test_patterns(metadata, max_files=None) + + # Convert to delete operations format + files_to_delete = [] + for file_info in existing_files: + if os.path.exists(file_info["real_path"]): + files_to_delete.append({ + "path": file_info["path"], + "real_path": file_info["real_path"], + "action": "delete", + "reason": "clean_before_restore" + }) + + return files_to_delete + except Exception: + # If pattern testing fails, return empty list to avoid breaking restore + return [] diff --git a/helpers/backup.py.dox.md b/helpers/backup.py.dox.md index 9ec127da51dcd41eb17a25ff770deeb625666a77..5cb9c24ec6d27df9fc88dcf634bffb72d564777c 100644 --- a/helpers/backup.py.dox.md +++ b/helpers/backup.py.dox.md @@ -1,3 +1,53 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:6a9081041e1d14aa0c78eb5705a05fc8c4f3df395a0c6a6418eeea3a8a23a3f2 -size 3948 +# backup.py DOX + +## Purpose + +- Own the `backup.py` helper module. +- This module builds, inspects, previews, tests, and restores Agent Zero backup archives. +- Keep this file-level DOX profile synchronized with `backup.py` because this directory is intentionally flat. + +## Ownership + +- `backup.py` owns the runtime implementation. +- `backup.py.dox.md` owns durable notes about responsibilities, contracts, side effects, and verification for that implementation. +- Classes: +- `BackupService` (no explicit base class) + - `get_default_backup_metadata(self) -> Dict[str, Any]` + - `async test_patterns(self, metadata: Dict[str, Any], max_files: Optional[int]=...) -> List[Dict[str, Any]]` + - `async create_backup(self, include_patterns: List[str], exclude_patterns: List[str], include_hidden: bool=..., backup_name: str=...) -> str` + - `async inspect_backup(self, backup_file) -> Dict[str, Any]` + - `async preview_restore(self, backup_file, restore_include_patterns: Optional[List[str]]=..., restore_exclude_patterns: Optional[List[str]]=..., overwrite_policy: str=..., clean_before_restore: bool=..., user_edited_metadata: Optional[Dict[str, Any]]=...) -> Dict[str, Any]` + - `async restore_backup(self, backup_file, restore_include_patterns: Optional[List[str]]=..., restore_exclude_patterns: Optional[List[str]]=..., overwrite_policy: str=..., clean_before_restore: bool=..., user_edited_metadata: Optional[Dict[str, Any]]=...) -> Dict[str, Any]` + +## Runtime Contracts + +- Helper modules own reusable framework APIs and must preserve public callers unless all callers, tests, and docs are updated together. +- Update this file whenever public functions, classes, persistence behavior, path/security assumptions, side effects, or cross-module contracts change. +- Observed side-effect areas: filesystem reads, filesystem writes, filesystem deletion, settings/state persistence, secret handling. +- Imported dependency areas include: `datetime`, `helpers`, `helpers.localization`, `helpers.print_style`, `json`, `os`, `pathspec`, `platform`, `tempfile`, `typing`, `zipfile`. +- `test_patterns(..., max_files=None)` is the unlimited scan mode. UI preview and dry-run callers may pass bounded limits, but real backup creation and restore clean-before-restore must use unlimited matching so archives and cleanup are not silently truncated. +- Default backup metadata includes persistent `/usr` data but excludes Time Travel shadow history under `usr/.time_travel/**`. +- Restoring `usr/.env` preserves the destination instance's allowed origins while restoring authentication and other portable configuration from the archive. + +## Key Concepts + +- Important called helpers/classes observed in the source: `self._get_agent_zero_version`, `files.get_abs_path`, `Localization.get.now_iso`, `self._get_default_patterns`, `self._parse_patterns`, `self.agent_zero_root.rstrip`, `patterns.split`, `join`, `file_path.lstrip`, `backed_up_agent_root.rstrip`, `current_agent_root.rstrip`, `self._patterns_to_string`, `self._get_explicit_patterns`, `tempfile.mkdtemp`, `os.path.join`, `self._translate_patterns`, `git.get_git_info`, `line.strip`, `line.startswith`, `getpass.getuser`. +- Keep request/response, tool, or helper semantics documented here at the same time as source changes. + +## Work Guidance + +- Preserve public helper APIs used by core code and plugins unless every caller is updated. +- Keep path, auth, secret, persistence, network, and subprocess behavior explicit and bounded. +- Prefer adding cohesive helper functions here only when behavior is reused across modules. + +## Verification + +- Run targeted tests for changed helper behavior; run security regressions for auth, filesystem, WebSocket, tunnel, upload, or secret-handling helpers. +- Related tests observed by source search: + - `tests/test_download_toast_regressions.py` + - `tests/test_office_document_store.py` + - `tests/test_self_update_tag_filter.py` + +## Child DOX Index + +No child DOX files. diff --git a/helpers/browser.py b/helpers/browser.py index 767561da7e5bbd4143e52e5b7a23d09d517d7e10..a10c4429e9261addd3f9ee2d391a7d2fd5238bc0 100644 --- a/helpers/browser.py +++ b/helpers/browser.py @@ -1,3 +1,385 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:81c642e0ecdcd7f66b2b1109b51e90cfc1b0e873705925f0ab9fc21b219755e1 -size 13988 +# import asyncio +# import re +# from bs4 import BeautifulSoup +# from playwright.async_api import ( +# async_playwright, +# Browser as PlaywrightBrowser, +# Page, +# Frame, +# BrowserContext, +# ) + +# from helpers import files + + +# class NoPageError(Exception): +# pass + + +# class Browser: + +# load_timeout = 10000 +# interact_timeout = 3000 +# selector_name = "data-a0sel3ct0r" + +# def __init__(self, headless=True): +# self.browser: PlaywrightBrowser = None # type: ignore +# self.context: BrowserContext = None # type: ignore +# self.page: Page = None # type: ignore +# self._playwright = None +# self.headless = headless +# self.contexts = {} +# self.last_selector = "" +# self.page_loaded = False +# self.navigation_count = 0 + +# async def __aenter__(self): +# await self.start() +# return self + +# async def __aexit__(self, exc_type, exc_val, exc_tb): +# await self.close() + +# async def start(self): +# """Start browser session""" +# self._playwright = await async_playwright().start() +# if not self.browser: +# self.browser = await self._playwright.chromium.launch( +# headless=self.headless, args=["--disable-http2"] +# ) +# if not self.context: +# self.context = await self.browser.new_context( +# user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.6422.141 Safari/537.36" +# ) + +# self.page = await self.context.new_page() +# await self.page.set_viewport_size({"width": 1200, "height": 1200}) + +# # Inject the JavaScript to modify the attachShadow method +# js_override = files.read_file("lib/browser/init_override.js") +# await self.page.add_init_script(js_override) + +# # Setup frame handling +# async def inject_script_into_frames(frame): +# try: +# await self.wait_tick() +# if not frame.is_detached(): +# async with asyncio.timeout(0.25): +# await frame.evaluate(js_override) +# print(f"Injected script into frame: {frame.url[:100]}") +# except Exception as e: +# # Frame might have been detached during injection, which is normal +# print( +# f"Could not inject into frame (possibly detached): {str(e)[:100]}" +# ) + +# self.page.on( +# "frameattached", +# lambda frame: asyncio.ensure_future(inject_script_into_frames(frame)), +# ) + +# # Handle page navigation events +# async def handle_navigation(frame): +# if frame == self.page.main_frame: +# print(f"Page navigated to: {frame.url[:100]}") +# self.page_loaded = False +# self.navigation_count += 1 + +# async def handle_load(dummy): +# print("Page load completed") +# self.page_loaded = True + +# async def handle_request(request): +# if ( +# request.is_navigation_request() +# and request.frame == self.page.main_frame +# ): +# print(f"Navigation started to: {request.url[:100]}") +# self.page_loaded = False +# self.navigation_count += 1 + +# self.page.on("request", handle_request) +# self.page.on("framenavigated", handle_navigation) +# self.page.on("load", handle_load) + +# async def close(self): +# """Close browser session""" +# if self.browser: +# await self.browser.close() +# if self._playwright: +# await self._playwright.stop() + +# async def open(self, url: str): +# """Open a URL in the browser""" +# self.last_selector = "" +# self.contexts = {} +# if self.page: +# await self.page.close() +# await self.start() +# try: +# await self.page.goto( +# url, wait_until="networkidle", timeout=Browser.load_timeout +# ) +# except TimeoutError as e: +# pass +# except Exception as e: +# print(f"Error opening page: {e}") +# raise e +# await self.wait_tick() + +# async def get_full_dom(self) -> str: +# """Get full DOM with unique selectors""" +# await self._check_page() +# js_code = files.read_file("lib/browser/extract_dom.js") + +# # Get all frames +# self.contexts = {} +# frame_contents = {} + +# # Extract content from each frame +# i = -1 +# for frame in self.page.frames: +# try: +# if frame.url: # and frame != self.page.main_frame: +# i += 1 +# frame_mark = self._num_to_alpha(i) + +# # Check if frame is still valid +# await self.wait_tick() +# if not frame.is_detached(): +# try: +# # short timeout to identify and skip unresponsive frames +# async with asyncio.timeout(0.25): +# await frame.evaluate("window.location.href") +# except TimeoutError as e: +# print(f"Skipping unresponsive frame: {frame.url}") +# continue + +# await frame.wait_for_load_state( +# "domcontentloaded", timeout=1000 +# ) + +# async with asyncio.timeout(1): +# content = await frame.evaluate( +# js_code, [frame_mark, self.selector_name] +# ) +# self.contexts[frame_mark] = frame +# frame_contents[frame.url] = content +# else: +# print(f"Warning: Frame was detached: {frame.url}") +# except Exception as e: +# print(f"Error extracting from frame {frame.url}: {e}") + +# # # Get main frame content +# # main_mark = self._num_to_alpha(0) +# # main_content = "" +# # try: +# # async with asyncio.timeout(1): +# # main_content = await self.page.evaluate(js_code, [main_mark, self.selector_name]) +# # self.contexts[main_mark] = self.page +# # except Exception as e: +# # print(f"Error when extracting from main frame: {e}") + +# # Replace iframe placeholders with actual content +# # for url, content in frame_contents.items(): +# # placeholder = f'