Spaces:
Running
Running
Upload folder using huggingface_hub
Browse filesThis view is limited to 50 files because it contains too many changes.
See raw diff
- Dockerfile +35 -0
- README.md +241 -6
- __init__.py +18 -0
- client.py +117 -0
- docker-compose.gitea.yml +49 -0
- envs/git_env/README.md +229 -0
- envs/git_env/__init__.py +18 -0
- envs/git_env/client.py +117 -0
- envs/git_env/docker-compose.gitea.yml +49 -0
- envs/git_env/models.py +72 -0
- envs/git_env/server/Dockerfile +33 -0
- envs/git_env/server/__init__.py +0 -0
- envs/git_env/server/app.py +67 -0
- envs/git_env/server/git_task_environment.py +282 -0
- models.py +72 -0
- pyproject.toml +114 -0
- server/Dockerfile +33 -0
- server/__init__.py +0 -0
- server/app.py +67 -0
- server/git_task_environment.py +282 -0
- src/__init__.py +7 -0
- src/openenv.egg-info/PKG-INFO +337 -0
- src/openenv.egg-info/SOURCES.txt +142 -0
- src/openenv.egg-info/dependency_links.txt +1 -0
- src/openenv.egg-info/entry_points.txt +2 -0
- src/openenv.egg-info/requires.txt +32 -0
- src/openenv.egg-info/top_level.txt +2 -0
- src/openenv/__init__.py +23 -0
- src/openenv/auto/__init__.py +39 -0
- src/openenv/auto/_discovery.py +584 -0
- src/openenv/auto/auto_action.py +276 -0
- src/openenv/auto/auto_env.py +896 -0
- src/openenv/cli/__init__.py +9 -0
- src/openenv/cli/__main__.py +62 -0
- src/openenv/cli/_cli_utils.py +79 -0
- src/openenv/cli/_validation.py +162 -0
- src/openenv/cli/commands/__init__.py +11 -0
- src/openenv/cli/commands/build.py +461 -0
- src/openenv/cli/commands/fork.py +197 -0
- src/openenv/cli/commands/init.py +500 -0
- src/openenv/cli/commands/push.py +718 -0
- src/openenv/cli/commands/serve.py +94 -0
- src/openenv/cli/commands/validate.py +108 -0
- src/openenv/cli/templates/__init__.py +7 -0
- src/openenv/cli/templates/openenv_env/.dockerignore +15 -0
- src/openenv/cli/templates/openenv_env/README.md +255 -0
- src/openenv/cli/templates/openenv_env/__init__.py +16 -0
- src/openenv/cli/templates/openenv_env/client.py +99 -0
- src/openenv/cli/templates/openenv_env/models.py +28 -0
- src/openenv/cli/templates/openenv_env/openenv.yaml +7 -0
Dockerfile
ADDED
|
@@ -0,0 +1,35 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Dockerfile for Git Environment
|
| 2 |
+
# Connects to an external shared Gitea service for task-based isolation
|
| 3 |
+
# Optimized for fast resets and minimal resource usage
|
| 4 |
+
|
| 5 |
+
# Use the standard openenv base image
|
| 6 |
+
ARG BASE_IMAGE=ghcr.io/meta-pytorch/openenv-base:latest
|
| 7 |
+
FROM ghcr.io/meta-pytorch/openenv-base:latest
|
| 8 |
+
|
| 9 |
+
# Install git and curl (no Gitea binary needed - connects to external service)
|
| 10 |
+
RUN apt-get update && apt-get install -y \
|
| 11 |
+
git \
|
| 12 |
+
curl \
|
| 13 |
+
ca-certificates \
|
| 14 |
+
&& rm -rf /var/lib/apt/lists/*
|
| 15 |
+
|
| 16 |
+
# Create workspace directory for git operations
|
| 17 |
+
RUN mkdir -p /workspace && chmod 777 /workspace
|
| 18 |
+
|
| 19 |
+
# Copy core and environment code
|
| 20 |
+
COPY src/core/ /app/src/core/
|
| 21 |
+
COPY envs/git_env/ /app/envs/git_env/
|
| 22 |
+
|
| 23 |
+
# Environment variables for Gitea connection
|
| 24 |
+
# These MUST be provided at runtime via -e flags or --env-file
|
| 25 |
+
# See .env.example for required variables
|
| 26 |
+
ENV WORKSPACE_DIR=/workspace
|
| 27 |
+
|
| 28 |
+
# Health check
|
| 29 |
+
HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \
|
| 30 |
+
CMD curl -f http://localhost:8000/health || exit 1
|
| 31 |
+
|
| 32 |
+
# Run the FastAPI server
|
| 33 |
+
CMD ["uvicorn", "envs.git_env.server.app:app", "--host", "0.0.0.0", "--port", "8000"]
|
| 34 |
+
|
| 35 |
+
ENV ENABLE_WEB_INTERFACE=true
|
README.md
CHANGED
|
@@ -1,10 +1,245 @@
|
|
| 1 |
---
|
| 2 |
-
title:
|
| 3 |
-
emoji: 📈
|
| 4 |
-
colorFrom: green
|
| 5 |
-
colorTo: pink
|
| 6 |
sdk: docker
|
| 7 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 8 |
---
|
| 9 |
|
| 10 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
---
|
| 2 |
+
title: git_env Environment
|
|
|
|
|
|
|
|
|
|
| 3 |
sdk: docker
|
| 4 |
+
app_port: 8000
|
| 5 |
+
base_path: /web
|
| 6 |
+
tags:
|
| 7 |
+
- openenv
|
| 8 |
+
- openenv-v2.1.0
|
| 9 |
---
|
| 10 |
|
| 11 |
+
# git_env Environment
|
| 12 |
+
|
| 13 |
+
Space URL: `https://huggingface.co/spaces/openenv/git_env-v2-1-0`
|
| 14 |
+
|
| 15 |
+
OpenEnv pinned ref: `v2.1.0`
|
| 16 |
+
|
| 17 |
+
# Git Environment
|
| 18 |
+
|
| 19 |
+
A Git server environment using Gitea that provides isolated Git repository management optimized for task-based RL training. Perfect for training agents on Git operations with fast reset capabilities.
|
| 20 |
+
|
| 21 |
+
## Overview
|
| 22 |
+
|
| 23 |
+
The Git Environment connects to a **shared external Gitea service** for optimal task-based isolation. **Perfect for**: RL training, task-based workflows, parallel execution
|
| 24 |
+
|
| 25 |
+
### Architecture
|
| 26 |
+
|
| 27 |
+
```
|
| 28 |
+
┌────────────────────────────────────┐
|
| 29 |
+
│ Shared Gitea (start once) │
|
| 30 |
+
│ Port 3000 │
|
| 31 |
+
│ - Pre-migrated repositories │
|
| 32 |
+
└──────────────┬─────────────────────┘
|
| 33 |
+
│ HTTP API
|
| 34 |
+
┾────────┼────────┾
|
| 35 |
+
│ │ │
|
| 36 |
+
┌───▼──┐ ┌──▼───┐ ┌──▼───┐
|
| 37 |
+
│Env 1 │ │Env 2 │ │Env 3 │
|
| 38 |
+
│Task A│ │Task B│ │Task A│
|
| 39 |
+
│@abc │ │@def │ │@abc │
|
| 40 |
+
└──────┘ └──────┘ └──────┘
|
| 41 |
+
Isolated workspaces
|
| 42 |
+
```
|
| 43 |
+
|
| 44 |
+
## Quick Start
|
| 45 |
+
|
| 46 |
+
```python
|
| 47 |
+
from envs.git_env import GitAction, GitEnv
|
| 48 |
+
|
| 49 |
+
# Create environment from Docker image
|
| 50 |
+
git_env = GitEnv.from_docker_image("git-env:latest")
|
| 51 |
+
|
| 52 |
+
# Reset environment
|
| 53 |
+
result = git_env.reset()
|
| 54 |
+
print(result.observation.message)
|
| 55 |
+
|
| 56 |
+
# List available repositories (pre-migrated to shared Gitea)
|
| 57 |
+
result = git_env.step(GitAction(action_type="list_repos"))
|
| 58 |
+
for repo in result.observation.repos:
|
| 59 |
+
print(f"{repo['name']}: {repo['clone_url']}")
|
| 60 |
+
|
| 61 |
+
# Clone to workspace
|
| 62 |
+
result = git_env.step(GitAction(action_type="clone_repo", repo_name="OpenEnv"))
|
| 63 |
+
print(result.observation.output) # Cloned to: /workspace/OpenEnv
|
| 64 |
+
|
| 65 |
+
# Execute git commands
|
| 66 |
+
result = git_env.step(GitAction(
|
| 67 |
+
action_type="execute_git_command",
|
| 68 |
+
command="status",
|
| 69 |
+
working_dir="OpenEnv"
|
| 70 |
+
))
|
| 71 |
+
print(result.observation.output)
|
| 72 |
+
|
| 73 |
+
# Cleanup
|
| 74 |
+
git_env.close()
|
| 75 |
+
```
|
| 76 |
+
|
| 77 |
+
## Setup and Running the Example
|
| 78 |
+
|
| 79 |
+
Complete setup (run these steps in order):
|
| 80 |
+
|
| 81 |
+
```bash
|
| 82 |
+
# 0. Configure environment variables
|
| 83 |
+
cp .env.example .env
|
| 84 |
+
# Edit .env and set your Gitea credentials if needed
|
| 85 |
+
|
| 86 |
+
# 1. Start shared Gitea service (one-time)
|
| 87 |
+
./scripts/setup_shared_gitea.sh
|
| 88 |
+
|
| 89 |
+
# 2. Migrate a test repository to Gitea (one-time)
|
| 90 |
+
docker exec openenv-gitea curl -X POST \
|
| 91 |
+
http://localhost:3000/api/v1/repos/migrate \
|
| 92 |
+
-u gitea:gitea123 \
|
| 93 |
+
-H 'Content-Type: application/json' \
|
| 94 |
+
-d '{
|
| 95 |
+
"clone_addr": "https://github.com/meta-pytorch/OpenEnv",
|
| 96 |
+
"repo_name": "OpenEnv",
|
| 97 |
+
"repo_owner": "gitea",
|
| 98 |
+
"service": "github"
|
| 99 |
+
}'
|
| 100 |
+
|
| 101 |
+
# 3. Build Docker images
|
| 102 |
+
docker build -t openenv-base:latest -f src/openenv/core/containers/images/Dockerfile .
|
| 103 |
+
docker build -t git-env:latest -f envs/git_env/server/Dockerfile .
|
| 104 |
+
|
| 105 |
+
# 4. Install Python dependencies
|
| 106 |
+
uv pip install -e .
|
| 107 |
+
|
| 108 |
+
# 5. Run the example (loads credentials from .env)
|
| 109 |
+
python3 examples/local_git_env.py
|
| 110 |
+
```
|
| 111 |
+
|
| 112 |
+
**Note**:
|
| 113 |
+
- Steps 1-3 are one-time setup
|
| 114 |
+
- Make sure `.env` file exists with your Gitea credentials
|
| 115 |
+
- After initial setup, you only need step 5 to run the example
|
| 116 |
+
|
| 117 |
+
## Environment Details
|
| 118 |
+
|
| 119 |
+
### Actions
|
| 120 |
+
|
| 121 |
+
**GitAction**: Unified action class for all Git operations
|
| 122 |
+
|
| 123 |
+
```python
|
| 124 |
+
@dataclass
|
| 125 |
+
class GitAction(Action):
|
| 126 |
+
action_type: str # Operation type
|
| 127 |
+
repo_name: str # Repository name (for clone/execute)
|
| 128 |
+
target_dir: Optional[str] # Target directory (for clone)
|
| 129 |
+
command: str # Git command (for execute)
|
| 130 |
+
working_dir: str # Working directory (for execute)
|
| 131 |
+
```
|
| 132 |
+
|
| 133 |
+
**Supported action_type values:**
|
| 134 |
+
|
| 135 |
+
#### "clone_repo" - Clone repository to workspace
|
| 136 |
+
```python
|
| 137 |
+
GitAction(action_type="clone_repo", repo_name="OpenEnv")
|
| 138 |
+
GitAction(action_type="clone_repo", repo_name="OpenEnv", target_dir="custom-dir")
|
| 139 |
+
```
|
| 140 |
+
|
| 141 |
+
#### "list_repos" - List available repositories
|
| 142 |
+
```python
|
| 143 |
+
GitAction(action_type="list_repos")
|
| 144 |
+
```
|
| 145 |
+
|
| 146 |
+
#### "execute_git_command" - Execute git command
|
| 147 |
+
```python
|
| 148 |
+
GitAction(
|
| 149 |
+
action_type="execute_git_command",
|
| 150 |
+
command="status",
|
| 151 |
+
working_dir="OpenEnv"
|
| 152 |
+
)
|
| 153 |
+
```
|
| 154 |
+
|
| 155 |
+
### Observation
|
| 156 |
+
|
| 157 |
+
**GitObservation**: Contains results of Git operations
|
| 158 |
+
|
| 159 |
+
```python
|
| 160 |
+
@dataclass
|
| 161 |
+
class GitObservation(Observation):
|
| 162 |
+
success: bool # Whether operation succeeded
|
| 163 |
+
message: str # Human-readable message
|
| 164 |
+
output: str # Command output or detailed result
|
| 165 |
+
error: str # Error message if failed
|
| 166 |
+
repos: list[dict] # List of repositories (for list_repos)
|
| 167 |
+
```
|
| 168 |
+
|
| 169 |
+
### State
|
| 170 |
+
|
| 171 |
+
**GitState**: Tracks environment state
|
| 172 |
+
|
| 173 |
+
```python
|
| 174 |
+
@dataclass
|
| 175 |
+
class GitState(State):
|
| 176 |
+
episode_id: str # Unique episode identifier
|
| 177 |
+
step_count: int # Number of steps taken
|
| 178 |
+
gitea_ready: bool # Whether Gitea is accessible
|
| 179 |
+
workspace_path: str # Path to workspace directory
|
| 180 |
+
```
|
| 181 |
+
|
| 182 |
+
## Advanced: Task-Based Training
|
| 183 |
+
|
| 184 |
+
For RL training scenarios where you need fast resets to specific repository states, you can configure task-specific base states in the environment. This is done by setting environment variables before starting containers:
|
| 185 |
+
|
| 186 |
+
```bash
|
| 187 |
+
# Example: Configure tasks for your training setup
|
| 188 |
+
docker run \
|
| 189 |
+
-e GITEA_URL=http://host.docker.internal:3000 \
|
| 190 |
+
-e TASK_REPOS='{"bug_fix": ["my-repo", "abc123"], "feature": ["my-repo", "def456"]}' \
|
| 191 |
+
git-env:latest
|
| 192 |
+
```
|
| 193 |
+
|
| 194 |
+
Then in your training code, environments automatically reset to the configured state.
|
| 195 |
+
|
| 196 |
+
See [`examples/local_git_env.py`](../../../examples/local_git_env.py) for complete working example.
|
| 197 |
+
|
| 198 |
+
## Project Structure
|
| 199 |
+
|
| 200 |
+
```
|
| 201 |
+
git_env/
|
| 202 |
+
├── README.md # This file
|
| 203 |
+
├── __init__.py # Exports
|
| 204 |
+
├── models.py # Action, Observation, State definitions
|
| 205 |
+
├── client.py # GitEnv HTTP client
|
| 206 |
+
├── docker-compose.gitea.yml # Shared Gitea service
|
| 207 |
+
└── server/
|
| 208 |
+
├── __init__.py
|
| 209 |
+
├── git_task_environment.py # Task-optimized environment
|
| 210 |
+
├── app.py # FastAPI application
|
| 211 |
+
└── Dockerfile # Lightweight container image
|
| 212 |
+
```
|
| 213 |
+
|
| 214 |
+
## Troubleshooting
|
| 215 |
+
|
| 216 |
+
### Gitea Not Ready
|
| 217 |
+
|
| 218 |
+
If environment can't connect to Gitea:
|
| 219 |
+
1. Ensure Gitea is running: `docker ps | grep gitea`
|
| 220 |
+
2. Check Gitea URL in environment: `GITEA_URL=http://gitea:3000`
|
| 221 |
+
3. Verify network connectivity: `docker network ls | grep openenv`
|
| 222 |
+
|
| 223 |
+
### Repository Not Found
|
| 224 |
+
|
| 225 |
+
Ensure repository is migrated to Gitea:
|
| 226 |
+
```bash
|
| 227 |
+
# List repos
|
| 228 |
+
curl -u gitea:gitea123 http://localhost:3000/api/v1/user/repos
|
| 229 |
+
```
|
| 230 |
+
|
| 231 |
+
### Slow Clone/Reset
|
| 232 |
+
|
| 233 |
+
- First clone is slower (~5-10s) - downloads from Gitea
|
| 234 |
+
- Subsequent resets are fast (<1s) - just git operations
|
| 235 |
+
- Use task-based mode with `task_repos` for optimal performance
|
| 236 |
+
|
| 237 |
+
|
| 238 |
+
## Security Notes
|
| 239 |
+
|
| 240 |
+
- **Never commit `.env` file** - it contains credentials (already in .gitignore)
|
| 241 |
+
- Use `.env.example` as a template and create your own `.env`
|
| 242 |
+
- Gitea credentials are for local development only
|
| 243 |
+
- For production, use proper secret management (Docker secrets, k8s secrets, etc.)
|
| 244 |
+
- All workspaces are isolated per container
|
| 245 |
+
- Only public repositories supported (no private repo auth)
|
__init__.py
ADDED
|
@@ -0,0 +1,18 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Git Environment - Git server with Gitea support.
|
| 3 |
+
|
| 4 |
+
This environment connects to a shared Gitea service for task-based isolation,
|
| 5 |
+
allowing agents to clone repositories, execute git commands, and manage workspaces.
|
| 6 |
+
|
| 7 |
+
Note: Repository migration is done externally via Gitea API before environment use.
|
| 8 |
+
"""
|
| 9 |
+
|
| 10 |
+
from .client import GitEnv
|
| 11 |
+
from .models import GitAction, GitObservation, GitState
|
| 12 |
+
|
| 13 |
+
__all__ = [
|
| 14 |
+
"GitEnv",
|
| 15 |
+
"GitAction",
|
| 16 |
+
"GitObservation",
|
| 17 |
+
"GitState",
|
| 18 |
+
]
|
client.py
ADDED
|
@@ -0,0 +1,117 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""
|
| 3 |
+
GitEnv Client
|
| 4 |
+
-------------
|
| 5 |
+
Client-side wrapper for the Git environment server.
|
| 6 |
+
|
| 7 |
+
This client maintains a persistent WebSocket connection to the environment
|
| 8 |
+
server, enabling efficient multi-step interactions with lower latency.
|
| 9 |
+
"""
|
| 10 |
+
|
| 11 |
+
from __future__ import annotations
|
| 12 |
+
|
| 13 |
+
from typing import TYPE_CHECKING
|
| 14 |
+
|
| 15 |
+
from openenv.core.client_types import StepResult
|
| 16 |
+
from openenv.core.env_client import EnvClient
|
| 17 |
+
|
| 18 |
+
from .models import GitAction, GitObservation, GitState
|
| 19 |
+
|
| 20 |
+
if TYPE_CHECKING:
|
| 21 |
+
from openenv.core.containers.runtime import ContainerProvider
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
class GitEnv(EnvClient[GitAction, GitObservation, GitState]):
|
| 25 |
+
"""
|
| 26 |
+
Client for Git Environment with Gitea server.
|
| 27 |
+
|
| 28 |
+
This client maintains a persistent WebSocket connection to the environment
|
| 29 |
+
server, enabling efficient multi-step interactions for Git operations.
|
| 30 |
+
|
| 31 |
+
The environment connects to a shared external Gitea service. Repositories
|
| 32 |
+
must be pre-migrated to Gitea before use.
|
| 33 |
+
|
| 34 |
+
Example:
|
| 35 |
+
>>> # From Docker image
|
| 36 |
+
>>> client = GitEnv.from_docker_image("git-env:latest")
|
| 37 |
+
>>> try:
|
| 38 |
+
... result = client.reset()
|
| 39 |
+
...
|
| 40 |
+
... # List available repositories
|
| 41 |
+
... from envs.git_env import GitAction
|
| 42 |
+
... result = client.step(GitAction(action_type="list_repos"))
|
| 43 |
+
... print(result.observation.repos)
|
| 44 |
+
...
|
| 45 |
+
... # Clone repository to workspace
|
| 46 |
+
... result = client.step(GitAction(action_type="clone_repo", repo_name="OpenEnv"))
|
| 47 |
+
...
|
| 48 |
+
... # Execute git commands
|
| 49 |
+
... result = client.step(GitAction(
|
| 50 |
+
... action_type="execute_git_command",
|
| 51 |
+
... command="status",
|
| 52 |
+
... working_dir="OpenEnv"
|
| 53 |
+
... ))
|
| 54 |
+
... finally:
|
| 55 |
+
... client.close()
|
| 56 |
+
"""
|
| 57 |
+
|
| 58 |
+
def _step_payload(self, action: GitAction) -> dict:
|
| 59 |
+
"""
|
| 60 |
+
Convert action to payload for server's /step endpoint.
|
| 61 |
+
|
| 62 |
+
Args:
|
| 63 |
+
action: GitAction to send to server
|
| 64 |
+
|
| 65 |
+
Returns:
|
| 66 |
+
Dictionary payload for HTTP request
|
| 67 |
+
"""
|
| 68 |
+
# Convert action to dictionary
|
| 69 |
+
payload = {
|
| 70 |
+
"action_type": action.action_type,
|
| 71 |
+
}
|
| 72 |
+
|
| 73 |
+
# Add type-specific fields for supported actions
|
| 74 |
+
if hasattr(action, "repo_name"):
|
| 75 |
+
payload["repo_name"] = action.repo_name
|
| 76 |
+
if hasattr(action, "target_dir"):
|
| 77 |
+
payload["target_dir"] = action.target_dir
|
| 78 |
+
if hasattr(action, "command"):
|
| 79 |
+
payload["command"] = action.command
|
| 80 |
+
if hasattr(action, "working_dir"):
|
| 81 |
+
payload["working_dir"] = action.working_dir
|
| 82 |
+
|
| 83 |
+
return payload
|
| 84 |
+
|
| 85 |
+
def _parse_result(self, payload: dict) -> StepResult[GitObservation]:
|
| 86 |
+
"""
|
| 87 |
+
Parse server response into StepResult.
|
| 88 |
+
|
| 89 |
+
Args:
|
| 90 |
+
payload: JSON response from /step endpoint
|
| 91 |
+
|
| 92 |
+
Returns:
|
| 93 |
+
StepResult containing GitObservation
|
| 94 |
+
"""
|
| 95 |
+
obs = GitObservation(**payload["observation"])
|
| 96 |
+
return StepResult(
|
| 97 |
+
observation=obs,
|
| 98 |
+
reward=payload.get("reward"),
|
| 99 |
+
done=bool(payload.get("done", False)),
|
| 100 |
+
)
|
| 101 |
+
|
| 102 |
+
def _parse_state(self, payload: dict) -> GitState:
|
| 103 |
+
"""
|
| 104 |
+
Parse server response into GitState object.
|
| 105 |
+
|
| 106 |
+
Args:
|
| 107 |
+
payload: JSON response from /state endpoint
|
| 108 |
+
|
| 109 |
+
Returns:
|
| 110 |
+
GitState object with environment state
|
| 111 |
+
"""
|
| 112 |
+
return GitState(
|
| 113 |
+
episode_id=payload.get("episode_id"),
|
| 114 |
+
step_count=payload.get("step_count", 0),
|
| 115 |
+
gitea_ready=payload.get("gitea_ready", False),
|
| 116 |
+
workspace_path=payload.get("workspace_path", "/workspace"),
|
| 117 |
+
)
|
docker-compose.gitea.yml
ADDED
|
@@ -0,0 +1,49 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Docker Compose configuration for shared Gitea service
|
| 2 |
+
# This runs a single Gitea instance that can be shared by multiple
|
| 3 |
+
# Git environment containers for optimal task-based isolation.
|
| 4 |
+
#
|
| 5 |
+
# Usage:
|
| 6 |
+
# docker-compose -f docker-compose.gitea.yml up -d
|
| 7 |
+
#
|
| 8 |
+
# The Gitea service will be available at:
|
| 9 |
+
# - http://localhost:3000 (web interface)
|
| 10 |
+
# - http://gitea:3000 (from other containers on the same network)
|
| 11 |
+
|
| 12 |
+
version: '3.8'
|
| 13 |
+
|
| 14 |
+
services:
|
| 15 |
+
gitea:
|
| 16 |
+
image: gitea/gitea:1.24
|
| 17 |
+
container_name: openenv-gitea
|
| 18 |
+
hostname: gitea
|
| 19 |
+
environment:
|
| 20 |
+
- USER_UID=1000
|
| 21 |
+
- USER_GID=1000
|
| 22 |
+
- GITEA__database__DB_TYPE=sqlite3
|
| 23 |
+
- GITEA__database__PATH=/data/gitea/gitea.db
|
| 24 |
+
- GITEA__server__DOMAIN=gitea
|
| 25 |
+
- GITEA__server__HTTP_PORT=3000
|
| 26 |
+
- GITEA__server__ROOT_URL=http://gitea:3000/
|
| 27 |
+
- GITEA__server__OFFLINE_MODE=true
|
| 28 |
+
restart: unless-stopped
|
| 29 |
+
networks:
|
| 30 |
+
- openenv-network
|
| 31 |
+
ports:
|
| 32 |
+
- "3000:3000"
|
| 33 |
+
volumes:
|
| 34 |
+
- gitea-data:/data
|
| 35 |
+
healthcheck:
|
| 36 |
+
test: ["CMD", "curl", "-f", "http://localhost:3000/"]
|
| 37 |
+
interval: 10s
|
| 38 |
+
timeout: 5s
|
| 39 |
+
retries: 5
|
| 40 |
+
start_period: 30s
|
| 41 |
+
|
| 42 |
+
networks:
|
| 43 |
+
openenv-network:
|
| 44 |
+
name: openenv-network
|
| 45 |
+
driver: bridge
|
| 46 |
+
|
| 47 |
+
volumes:
|
| 48 |
+
gitea-data:
|
| 49 |
+
name: openenv-gitea-data
|
envs/git_env/README.md
ADDED
|
@@ -0,0 +1,229 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Git Environment
|
| 2 |
+
|
| 3 |
+
A Git server environment using Gitea that provides isolated Git repository management optimized for task-based RL training. Perfect for training agents on Git operations with fast reset capabilities.
|
| 4 |
+
|
| 5 |
+
## Overview
|
| 6 |
+
|
| 7 |
+
The Git Environment connects to a **shared external Gitea service** for optimal task-based isolation. **Perfect for**: RL training, task-based workflows, parallel execution
|
| 8 |
+
|
| 9 |
+
### Architecture
|
| 10 |
+
|
| 11 |
+
```
|
| 12 |
+
┌────────────────────────────────────┐
|
| 13 |
+
│ Shared Gitea (start once) │
|
| 14 |
+
│ Port 3000 │
|
| 15 |
+
│ - Pre-migrated repositories │
|
| 16 |
+
└──────────────┬─────────────────────┘
|
| 17 |
+
│ HTTP API
|
| 18 |
+
┾────────┼────────┾
|
| 19 |
+
│ │ │
|
| 20 |
+
┌───▼──┐ ┌──▼───┐ ┌──▼───┐
|
| 21 |
+
│Env 1 │ │Env 2 │ │Env 3 │
|
| 22 |
+
│Task A│ │Task B│ │Task A│
|
| 23 |
+
│@abc │ │@def │ │@abc │
|
| 24 |
+
└──────┘ └──────┘ └──────┘
|
| 25 |
+
Isolated workspaces
|
| 26 |
+
```
|
| 27 |
+
|
| 28 |
+
## Quick Start
|
| 29 |
+
|
| 30 |
+
```python
|
| 31 |
+
from envs.git_env import GitAction, GitEnv
|
| 32 |
+
|
| 33 |
+
# Create environment from Docker image
|
| 34 |
+
git_env = GitEnv.from_docker_image("git-env:latest")
|
| 35 |
+
|
| 36 |
+
# Reset environment
|
| 37 |
+
result = git_env.reset()
|
| 38 |
+
print(result.observation.message)
|
| 39 |
+
|
| 40 |
+
# List available repositories (pre-migrated to shared Gitea)
|
| 41 |
+
result = git_env.step(GitAction(action_type="list_repos"))
|
| 42 |
+
for repo in result.observation.repos:
|
| 43 |
+
print(f"{repo['name']}: {repo['clone_url']}")
|
| 44 |
+
|
| 45 |
+
# Clone to workspace
|
| 46 |
+
result = git_env.step(GitAction(action_type="clone_repo", repo_name="OpenEnv"))
|
| 47 |
+
print(result.observation.output) # Cloned to: /workspace/OpenEnv
|
| 48 |
+
|
| 49 |
+
# Execute git commands
|
| 50 |
+
result = git_env.step(GitAction(
|
| 51 |
+
action_type="execute_git_command",
|
| 52 |
+
command="status",
|
| 53 |
+
working_dir="OpenEnv"
|
| 54 |
+
))
|
| 55 |
+
print(result.observation.output)
|
| 56 |
+
|
| 57 |
+
# Cleanup
|
| 58 |
+
git_env.close()
|
| 59 |
+
```
|
| 60 |
+
|
| 61 |
+
## Setup and Running the Example
|
| 62 |
+
|
| 63 |
+
Complete setup (run these steps in order):
|
| 64 |
+
|
| 65 |
+
```bash
|
| 66 |
+
# 0. Configure environment variables
|
| 67 |
+
cp .env.example .env
|
| 68 |
+
# Edit .env and set your Gitea credentials if needed
|
| 69 |
+
|
| 70 |
+
# 1. Start shared Gitea service (one-time)
|
| 71 |
+
./scripts/setup_shared_gitea.sh
|
| 72 |
+
|
| 73 |
+
# 2. Migrate a test repository to Gitea (one-time)
|
| 74 |
+
docker exec openenv-gitea curl -X POST \
|
| 75 |
+
http://localhost:3000/api/v1/repos/migrate \
|
| 76 |
+
-u gitea:gitea123 \
|
| 77 |
+
-H 'Content-Type: application/json' \
|
| 78 |
+
-d '{
|
| 79 |
+
"clone_addr": "https://github.com/meta-pytorch/OpenEnv",
|
| 80 |
+
"repo_name": "OpenEnv",
|
| 81 |
+
"repo_owner": "gitea",
|
| 82 |
+
"service": "github"
|
| 83 |
+
}'
|
| 84 |
+
|
| 85 |
+
# 3. Build Docker images
|
| 86 |
+
docker build -t openenv-base:latest -f src/openenv/core/containers/images/Dockerfile .
|
| 87 |
+
docker build -t git-env:latest -f envs/git_env/server/Dockerfile .
|
| 88 |
+
|
| 89 |
+
# 4. Install Python dependencies
|
| 90 |
+
uv pip install -e .
|
| 91 |
+
|
| 92 |
+
# 5. Run the example (loads credentials from .env)
|
| 93 |
+
python3 examples/local_git_env.py
|
| 94 |
+
```
|
| 95 |
+
|
| 96 |
+
**Note**:
|
| 97 |
+
- Steps 1-3 are one-time setup
|
| 98 |
+
- Make sure `.env` file exists with your Gitea credentials
|
| 99 |
+
- After initial setup, you only need step 5 to run the example
|
| 100 |
+
|
| 101 |
+
## Environment Details
|
| 102 |
+
|
| 103 |
+
### Actions
|
| 104 |
+
|
| 105 |
+
**GitAction**: Unified action class for all Git operations
|
| 106 |
+
|
| 107 |
+
```python
|
| 108 |
+
@dataclass
|
| 109 |
+
class GitAction(Action):
|
| 110 |
+
action_type: str # Operation type
|
| 111 |
+
repo_name: str # Repository name (for clone/execute)
|
| 112 |
+
target_dir: Optional[str] # Target directory (for clone)
|
| 113 |
+
command: str # Git command (for execute)
|
| 114 |
+
working_dir: str # Working directory (for execute)
|
| 115 |
+
```
|
| 116 |
+
|
| 117 |
+
**Supported action_type values:**
|
| 118 |
+
|
| 119 |
+
#### "clone_repo" - Clone repository to workspace
|
| 120 |
+
```python
|
| 121 |
+
GitAction(action_type="clone_repo", repo_name="OpenEnv")
|
| 122 |
+
GitAction(action_type="clone_repo", repo_name="OpenEnv", target_dir="custom-dir")
|
| 123 |
+
```
|
| 124 |
+
|
| 125 |
+
#### "list_repos" - List available repositories
|
| 126 |
+
```python
|
| 127 |
+
GitAction(action_type="list_repos")
|
| 128 |
+
```
|
| 129 |
+
|
| 130 |
+
#### "execute_git_command" - Execute git command
|
| 131 |
+
```python
|
| 132 |
+
GitAction(
|
| 133 |
+
action_type="execute_git_command",
|
| 134 |
+
command="status",
|
| 135 |
+
working_dir="OpenEnv"
|
| 136 |
+
)
|
| 137 |
+
```
|
| 138 |
+
|
| 139 |
+
### Observation
|
| 140 |
+
|
| 141 |
+
**GitObservation**: Contains results of Git operations
|
| 142 |
+
|
| 143 |
+
```python
|
| 144 |
+
@dataclass
|
| 145 |
+
class GitObservation(Observation):
|
| 146 |
+
success: bool # Whether operation succeeded
|
| 147 |
+
message: str # Human-readable message
|
| 148 |
+
output: str # Command output or detailed result
|
| 149 |
+
error: str # Error message if failed
|
| 150 |
+
repos: list[dict] # List of repositories (for list_repos)
|
| 151 |
+
```
|
| 152 |
+
|
| 153 |
+
### State
|
| 154 |
+
|
| 155 |
+
**GitState**: Tracks environment state
|
| 156 |
+
|
| 157 |
+
```python
|
| 158 |
+
@dataclass
|
| 159 |
+
class GitState(State):
|
| 160 |
+
episode_id: str # Unique episode identifier
|
| 161 |
+
step_count: int # Number of steps taken
|
| 162 |
+
gitea_ready: bool # Whether Gitea is accessible
|
| 163 |
+
workspace_path: str # Path to workspace directory
|
| 164 |
+
```
|
| 165 |
+
|
| 166 |
+
## Advanced: Task-Based Training
|
| 167 |
+
|
| 168 |
+
For RL training scenarios where you need fast resets to specific repository states, you can configure task-specific base states in the environment. This is done by setting environment variables before starting containers:
|
| 169 |
+
|
| 170 |
+
```bash
|
| 171 |
+
# Example: Configure tasks for your training setup
|
| 172 |
+
docker run \
|
| 173 |
+
-e GITEA_URL=http://host.docker.internal:3000 \
|
| 174 |
+
-e TASK_REPOS='{"bug_fix": ["my-repo", "abc123"], "feature": ["my-repo", "def456"]}' \
|
| 175 |
+
git-env:latest
|
| 176 |
+
```
|
| 177 |
+
|
| 178 |
+
Then in your training code, environments automatically reset to the configured state.
|
| 179 |
+
|
| 180 |
+
See [`examples/local_git_env.py`](../../../examples/local_git_env.py) for complete working example.
|
| 181 |
+
|
| 182 |
+
## Project Structure
|
| 183 |
+
|
| 184 |
+
```
|
| 185 |
+
git_env/
|
| 186 |
+
├── README.md # This file
|
| 187 |
+
├── __init__.py # Exports
|
| 188 |
+
├── models.py # Action, Observation, State definitions
|
| 189 |
+
├── client.py # GitEnv HTTP client
|
| 190 |
+
├── docker-compose.gitea.yml # Shared Gitea service
|
| 191 |
+
└── server/
|
| 192 |
+
├── __init__.py
|
| 193 |
+
├── git_task_environment.py # Task-optimized environment
|
| 194 |
+
├── app.py # FastAPI application
|
| 195 |
+
└── Dockerfile # Lightweight container image
|
| 196 |
+
```
|
| 197 |
+
|
| 198 |
+
## Troubleshooting
|
| 199 |
+
|
| 200 |
+
### Gitea Not Ready
|
| 201 |
+
|
| 202 |
+
If environment can't connect to Gitea:
|
| 203 |
+
1. Ensure Gitea is running: `docker ps | grep gitea`
|
| 204 |
+
2. Check Gitea URL in environment: `GITEA_URL=http://gitea:3000`
|
| 205 |
+
3. Verify network connectivity: `docker network ls | grep openenv`
|
| 206 |
+
|
| 207 |
+
### Repository Not Found
|
| 208 |
+
|
| 209 |
+
Ensure repository is migrated to Gitea:
|
| 210 |
+
```bash
|
| 211 |
+
# List repos
|
| 212 |
+
curl -u gitea:gitea123 http://localhost:3000/api/v1/user/repos
|
| 213 |
+
```
|
| 214 |
+
|
| 215 |
+
### Slow Clone/Reset
|
| 216 |
+
|
| 217 |
+
- First clone is slower (~5-10s) - downloads from Gitea
|
| 218 |
+
- Subsequent resets are fast (<1s) - just git operations
|
| 219 |
+
- Use task-based mode with `task_repos` for optimal performance
|
| 220 |
+
|
| 221 |
+
|
| 222 |
+
## Security Notes
|
| 223 |
+
|
| 224 |
+
- **Never commit `.env` file** - it contains credentials (already in .gitignore)
|
| 225 |
+
- Use `.env.example` as a template and create your own `.env`
|
| 226 |
+
- Gitea credentials are for local development only
|
| 227 |
+
- For production, use proper secret management (Docker secrets, k8s secrets, etc.)
|
| 228 |
+
- All workspaces are isolated per container
|
| 229 |
+
- Only public repositories supported (no private repo auth)
|
envs/git_env/__init__.py
ADDED
|
@@ -0,0 +1,18 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Git Environment - Git server with Gitea support.
|
| 3 |
+
|
| 4 |
+
This environment connects to a shared Gitea service for task-based isolation,
|
| 5 |
+
allowing agents to clone repositories, execute git commands, and manage workspaces.
|
| 6 |
+
|
| 7 |
+
Note: Repository migration is done externally via Gitea API before environment use.
|
| 8 |
+
"""
|
| 9 |
+
|
| 10 |
+
from .client import GitEnv
|
| 11 |
+
from .models import GitAction, GitObservation, GitState
|
| 12 |
+
|
| 13 |
+
__all__ = [
|
| 14 |
+
"GitEnv",
|
| 15 |
+
"GitAction",
|
| 16 |
+
"GitObservation",
|
| 17 |
+
"GitState",
|
| 18 |
+
]
|
envs/git_env/client.py
ADDED
|
@@ -0,0 +1,117 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""
|
| 3 |
+
GitEnv Client
|
| 4 |
+
-------------
|
| 5 |
+
Client-side wrapper for the Git environment server.
|
| 6 |
+
|
| 7 |
+
This client maintains a persistent WebSocket connection to the environment
|
| 8 |
+
server, enabling efficient multi-step interactions with lower latency.
|
| 9 |
+
"""
|
| 10 |
+
|
| 11 |
+
from __future__ import annotations
|
| 12 |
+
|
| 13 |
+
from typing import TYPE_CHECKING
|
| 14 |
+
|
| 15 |
+
from openenv.core.client_types import StepResult
|
| 16 |
+
from openenv.core.env_client import EnvClient
|
| 17 |
+
|
| 18 |
+
from .models import GitAction, GitObservation, GitState
|
| 19 |
+
|
| 20 |
+
if TYPE_CHECKING:
|
| 21 |
+
from openenv.core.containers.runtime import ContainerProvider
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
class GitEnv(EnvClient[GitAction, GitObservation, GitState]):
|
| 25 |
+
"""
|
| 26 |
+
Client for Git Environment with Gitea server.
|
| 27 |
+
|
| 28 |
+
This client maintains a persistent WebSocket connection to the environment
|
| 29 |
+
server, enabling efficient multi-step interactions for Git operations.
|
| 30 |
+
|
| 31 |
+
The environment connects to a shared external Gitea service. Repositories
|
| 32 |
+
must be pre-migrated to Gitea before use.
|
| 33 |
+
|
| 34 |
+
Example:
|
| 35 |
+
>>> # From Docker image
|
| 36 |
+
>>> client = GitEnv.from_docker_image("git-env:latest")
|
| 37 |
+
>>> try:
|
| 38 |
+
... result = client.reset()
|
| 39 |
+
...
|
| 40 |
+
... # List available repositories
|
| 41 |
+
... from envs.git_env import GitAction
|
| 42 |
+
... result = client.step(GitAction(action_type="list_repos"))
|
| 43 |
+
... print(result.observation.repos)
|
| 44 |
+
...
|
| 45 |
+
... # Clone repository to workspace
|
| 46 |
+
... result = client.step(GitAction(action_type="clone_repo", repo_name="OpenEnv"))
|
| 47 |
+
...
|
| 48 |
+
... # Execute git commands
|
| 49 |
+
... result = client.step(GitAction(
|
| 50 |
+
... action_type="execute_git_command",
|
| 51 |
+
... command="status",
|
| 52 |
+
... working_dir="OpenEnv"
|
| 53 |
+
... ))
|
| 54 |
+
... finally:
|
| 55 |
+
... client.close()
|
| 56 |
+
"""
|
| 57 |
+
|
| 58 |
+
def _step_payload(self, action: GitAction) -> dict:
|
| 59 |
+
"""
|
| 60 |
+
Convert action to payload for server's /step endpoint.
|
| 61 |
+
|
| 62 |
+
Args:
|
| 63 |
+
action: GitAction to send to server
|
| 64 |
+
|
| 65 |
+
Returns:
|
| 66 |
+
Dictionary payload for HTTP request
|
| 67 |
+
"""
|
| 68 |
+
# Convert action to dictionary
|
| 69 |
+
payload = {
|
| 70 |
+
"action_type": action.action_type,
|
| 71 |
+
}
|
| 72 |
+
|
| 73 |
+
# Add type-specific fields for supported actions
|
| 74 |
+
if hasattr(action, "repo_name"):
|
| 75 |
+
payload["repo_name"] = action.repo_name
|
| 76 |
+
if hasattr(action, "target_dir"):
|
| 77 |
+
payload["target_dir"] = action.target_dir
|
| 78 |
+
if hasattr(action, "command"):
|
| 79 |
+
payload["command"] = action.command
|
| 80 |
+
if hasattr(action, "working_dir"):
|
| 81 |
+
payload["working_dir"] = action.working_dir
|
| 82 |
+
|
| 83 |
+
return payload
|
| 84 |
+
|
| 85 |
+
def _parse_result(self, payload: dict) -> StepResult[GitObservation]:
|
| 86 |
+
"""
|
| 87 |
+
Parse server response into StepResult.
|
| 88 |
+
|
| 89 |
+
Args:
|
| 90 |
+
payload: JSON response from /step endpoint
|
| 91 |
+
|
| 92 |
+
Returns:
|
| 93 |
+
StepResult containing GitObservation
|
| 94 |
+
"""
|
| 95 |
+
obs = GitObservation(**payload["observation"])
|
| 96 |
+
return StepResult(
|
| 97 |
+
observation=obs,
|
| 98 |
+
reward=payload.get("reward"),
|
| 99 |
+
done=bool(payload.get("done", False)),
|
| 100 |
+
)
|
| 101 |
+
|
| 102 |
+
def _parse_state(self, payload: dict) -> GitState:
|
| 103 |
+
"""
|
| 104 |
+
Parse server response into GitState object.
|
| 105 |
+
|
| 106 |
+
Args:
|
| 107 |
+
payload: JSON response from /state endpoint
|
| 108 |
+
|
| 109 |
+
Returns:
|
| 110 |
+
GitState object with environment state
|
| 111 |
+
"""
|
| 112 |
+
return GitState(
|
| 113 |
+
episode_id=payload.get("episode_id"),
|
| 114 |
+
step_count=payload.get("step_count", 0),
|
| 115 |
+
gitea_ready=payload.get("gitea_ready", False),
|
| 116 |
+
workspace_path=payload.get("workspace_path", "/workspace"),
|
| 117 |
+
)
|
envs/git_env/docker-compose.gitea.yml
ADDED
|
@@ -0,0 +1,49 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Docker Compose configuration for shared Gitea service
|
| 2 |
+
# This runs a single Gitea instance that can be shared by multiple
|
| 3 |
+
# Git environment containers for optimal task-based isolation.
|
| 4 |
+
#
|
| 5 |
+
# Usage:
|
| 6 |
+
# docker-compose -f docker-compose.gitea.yml up -d
|
| 7 |
+
#
|
| 8 |
+
# The Gitea service will be available at:
|
| 9 |
+
# - http://localhost:3000 (web interface)
|
| 10 |
+
# - http://gitea:3000 (from other containers on the same network)
|
| 11 |
+
|
| 12 |
+
version: '3.8'
|
| 13 |
+
|
| 14 |
+
services:
|
| 15 |
+
gitea:
|
| 16 |
+
image: gitea/gitea:1.24
|
| 17 |
+
container_name: openenv-gitea
|
| 18 |
+
hostname: gitea
|
| 19 |
+
environment:
|
| 20 |
+
- USER_UID=1000
|
| 21 |
+
- USER_GID=1000
|
| 22 |
+
- GITEA__database__DB_TYPE=sqlite3
|
| 23 |
+
- GITEA__database__PATH=/data/gitea/gitea.db
|
| 24 |
+
- GITEA__server__DOMAIN=gitea
|
| 25 |
+
- GITEA__server__HTTP_PORT=3000
|
| 26 |
+
- GITEA__server__ROOT_URL=http://gitea:3000/
|
| 27 |
+
- GITEA__server__OFFLINE_MODE=true
|
| 28 |
+
restart: unless-stopped
|
| 29 |
+
networks:
|
| 30 |
+
- openenv-network
|
| 31 |
+
ports:
|
| 32 |
+
- "3000:3000"
|
| 33 |
+
volumes:
|
| 34 |
+
- gitea-data:/data
|
| 35 |
+
healthcheck:
|
| 36 |
+
test: ["CMD", "curl", "-f", "http://localhost:3000/"]
|
| 37 |
+
interval: 10s
|
| 38 |
+
timeout: 5s
|
| 39 |
+
retries: 5
|
| 40 |
+
start_period: 30s
|
| 41 |
+
|
| 42 |
+
networks:
|
| 43 |
+
openenv-network:
|
| 44 |
+
name: openenv-network
|
| 45 |
+
driver: bridge
|
| 46 |
+
|
| 47 |
+
volumes:
|
| 48 |
+
gitea-data:
|
| 49 |
+
name: openenv-gitea-data
|
envs/git_env/models.py
ADDED
|
@@ -0,0 +1,72 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
|
| 3 |
+
"""
|
| 4 |
+
envs/git_env/models.py
|
| 5 |
+
--------------------------------
|
| 6 |
+
Action/Observation types for the Git environment with Gitea server.
|
| 7 |
+
"""
|
| 8 |
+
|
| 9 |
+
from __future__ import annotations
|
| 10 |
+
|
| 11 |
+
from pydantic import Field
|
| 12 |
+
from typing import Optional
|
| 13 |
+
|
| 14 |
+
from openenv.core.env_server import Action, Observation, State
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
class GitAction(Action):
|
| 18 |
+
"""
|
| 19 |
+
Action for Git environment operations.
|
| 20 |
+
|
| 21 |
+
This unified action class supports multiple operation types:
|
| 22 |
+
- clone_repo: Clone a repository from Gitea to workspace
|
| 23 |
+
- list_repos: List all available repositories
|
| 24 |
+
- execute_git_command: Execute a git command in workspace
|
| 25 |
+
|
| 26 |
+
Attributes:
|
| 27 |
+
action_type: Type of operation ("clone_repo", "list_repos", "execute_git_command")
|
| 28 |
+
repo_name: Name of repository (for clone_repo, execute_git_command)
|
| 29 |
+
target_dir: Target directory for clone (optional)
|
| 30 |
+
command: Git command to execute (for execute_git_command)
|
| 31 |
+
working_dir: Working directory relative to workspace (for execute_git_command)
|
| 32 |
+
"""
|
| 33 |
+
|
| 34 |
+
action_type: str = "list_repos"
|
| 35 |
+
repo_name: str = ""
|
| 36 |
+
target_dir: Optional[str] = None
|
| 37 |
+
command: str = ""
|
| 38 |
+
working_dir: str = ""
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
class GitObservation(Observation):
|
| 42 |
+
"""
|
| 43 |
+
Result of executing a Git action.
|
| 44 |
+
|
| 45 |
+
Attributes:
|
| 46 |
+
success: Whether the action was successful
|
| 47 |
+
message: Human-readable message about the result
|
| 48 |
+
output: Command output or detailed result
|
| 49 |
+
error: Error message if action failed
|
| 50 |
+
repos: List of repositories (for list_repos action)
|
| 51 |
+
"""
|
| 52 |
+
|
| 53 |
+
success: bool = False
|
| 54 |
+
message: str = ""
|
| 55 |
+
output: str = ""
|
| 56 |
+
error: str = ""
|
| 57 |
+
repos: list[dict[str, str]] = Field(default_factory=list)
|
| 58 |
+
|
| 59 |
+
|
| 60 |
+
class GitState(State):
|
| 61 |
+
"""
|
| 62 |
+
State for Git environment.
|
| 63 |
+
|
| 64 |
+
Attributes:
|
| 65 |
+
episode_id: Unique identifier for the episode
|
| 66 |
+
step_count: Number of steps taken
|
| 67 |
+
gitea_ready: Whether Gitea server is accessible
|
| 68 |
+
workspace_path: Path to the workspace directory
|
| 69 |
+
"""
|
| 70 |
+
|
| 71 |
+
gitea_ready: bool = False
|
| 72 |
+
workspace_path: str = "/workspace"
|
envs/git_env/server/Dockerfile
ADDED
|
@@ -0,0 +1,33 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Dockerfile for Git Environment
|
| 2 |
+
# Connects to an external shared Gitea service for task-based isolation
|
| 3 |
+
# Optimized for fast resets and minimal resource usage
|
| 4 |
+
|
| 5 |
+
# Use the standard openenv base image
|
| 6 |
+
ARG BASE_IMAGE=openenv-base:latest
|
| 7 |
+
FROM ${BASE_IMAGE}
|
| 8 |
+
|
| 9 |
+
# Install git and curl (no Gitea binary needed - connects to external service)
|
| 10 |
+
RUN apt-get update && apt-get install -y \
|
| 11 |
+
git \
|
| 12 |
+
curl \
|
| 13 |
+
ca-certificates \
|
| 14 |
+
&& rm -rf /var/lib/apt/lists/*
|
| 15 |
+
|
| 16 |
+
# Create workspace directory for git operations
|
| 17 |
+
RUN mkdir -p /workspace && chmod 777 /workspace
|
| 18 |
+
|
| 19 |
+
# Copy core and environment code
|
| 20 |
+
COPY src/core/ /app/src/core/
|
| 21 |
+
COPY envs/git_env/ /app/envs/git_env/
|
| 22 |
+
|
| 23 |
+
# Environment variables for Gitea connection
|
| 24 |
+
# These MUST be provided at runtime via -e flags or --env-file
|
| 25 |
+
# See .env.example for required variables
|
| 26 |
+
ENV WORKSPACE_DIR=/workspace
|
| 27 |
+
|
| 28 |
+
# Health check
|
| 29 |
+
HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \
|
| 30 |
+
CMD curl -f http://localhost:8000/health || exit 1
|
| 31 |
+
|
| 32 |
+
# Run the FastAPI server
|
| 33 |
+
CMD ["uvicorn", "envs.git_env.server.app:app", "--host", "0.0.0.0", "--port", "8000"]
|
envs/git_env/server/__init__.py
ADDED
|
File without changes
|
envs/git_env/server/app.py
ADDED
|
@@ -0,0 +1,67 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
|
| 3 |
+
"""
|
| 4 |
+
FastAPI application for Git Environment.
|
| 5 |
+
|
| 6 |
+
This module creates an HTTP server for the Git environment that connects
|
| 7 |
+
to a shared external Gitea service for fast, isolated task resets.
|
| 8 |
+
|
| 9 |
+
Environment variables (required):
|
| 10 |
+
GITEA_URL: URL of shared Gitea service
|
| 11 |
+
GITEA_USERNAME: Gitea username
|
| 12 |
+
GITEA_PASSWORD: Gitea password
|
| 13 |
+
WORKSPACE_DIR: Workspace directory (optional, default: /workspace)
|
| 14 |
+
|
| 15 |
+
Usage:
|
| 16 |
+
# Development (with auto-reload):
|
| 17 |
+
uvicorn envs.git_env.server.app:app --reload --host 0.0.0.0 --port 8000
|
| 18 |
+
|
| 19 |
+
# Production:
|
| 20 |
+
uvicorn envs.git_env.server.app:app --host 0.0.0.0 --port 8000 --workers 4
|
| 21 |
+
|
| 22 |
+
# With custom Gitea:
|
| 23 |
+
GITEA_URL=http://my-gitea:3000 uvicorn envs.git_env.server.app:app --host 0.0.0.0 --port 8000
|
| 24 |
+
"""
|
| 25 |
+
|
| 26 |
+
import os
|
| 27 |
+
|
| 28 |
+
from openenv.core.env_server import create_app
|
| 29 |
+
|
| 30 |
+
from ..models import GitAction, GitObservation
|
| 31 |
+
from .git_task_environment import GitTaskEnvironment
|
| 32 |
+
|
| 33 |
+
# Read configuration from environment variables
|
| 34 |
+
gitea_url = os.getenv("GITEA_URL")
|
| 35 |
+
gitea_username = os.getenv("GITEA_USERNAME")
|
| 36 |
+
gitea_password = os.getenv("GITEA_PASSWORD")
|
| 37 |
+
workspace_dir = os.getenv("WORKSPACE_DIR", "/workspace")
|
| 38 |
+
|
| 39 |
+
# Validate required environment variables
|
| 40 |
+
if not gitea_url:
|
| 41 |
+
raise RuntimeError("GITEA_URL environment variable is required")
|
| 42 |
+
if not gitea_username:
|
| 43 |
+
raise RuntimeError("GITEA_USERNAME environment variable is required")
|
| 44 |
+
if not gitea_password:
|
| 45 |
+
raise RuntimeError("GITEA_PASSWORD environment variable is required")
|
| 46 |
+
|
| 47 |
+
|
| 48 |
+
# Factory function to create GitTaskEnvironment instances
|
| 49 |
+
def create_git_environment():
|
| 50 |
+
"""Factory function that creates GitTaskEnvironment with config."""
|
| 51 |
+
return GitTaskEnvironment(
|
| 52 |
+
gitea_url=gitea_url,
|
| 53 |
+
username=gitea_username,
|
| 54 |
+
password=gitea_password,
|
| 55 |
+
workspace_dir=workspace_dir,
|
| 56 |
+
)
|
| 57 |
+
|
| 58 |
+
|
| 59 |
+
# Create the app with web interface and README integration
|
| 60 |
+
# Pass the factory function instead of an instance for WebSocket session support
|
| 61 |
+
app = create_app(create_git_environment, GitAction, GitObservation, env_name="git_env")
|
| 62 |
+
|
| 63 |
+
|
| 64 |
+
if __name__ == "__main__":
|
| 65 |
+
import uvicorn
|
| 66 |
+
|
| 67 |
+
uvicorn.run(app, host="0.0.0.0", port=8000)
|
envs/git_env/server/git_task_environment.py
ADDED
|
@@ -0,0 +1,282 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
|
| 3 |
+
"""
|
| 4 |
+
Git Task Environment - Optimized for task-based isolation.
|
| 5 |
+
|
| 6 |
+
This module provides an optimized Git environment for scenarios where:
|
| 7 |
+
- Multiple tasks share the same base repository states
|
| 8 |
+
- Tasks need fast reset() to reproducible states
|
| 9 |
+
- Each task has an isolated workspace
|
| 10 |
+
- A shared Gitea service provides repository storage
|
| 11 |
+
"""
|
| 12 |
+
|
| 13 |
+
import uuid
|
| 14 |
+
|
| 15 |
+
from openenv.core.env_server import Action, Environment, Observation
|
| 16 |
+
from openenv.core.tools import GitServerClient
|
| 17 |
+
|
| 18 |
+
from ..models import GitAction, GitObservation, GitState
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
class GitTaskEnvironment(Environment):
|
| 22 |
+
"""
|
| 23 |
+
Git Environment optimized for task-based isolation.
|
| 24 |
+
|
| 25 |
+
This environment connects to a shared Gitea service and provides:
|
| 26 |
+
- Fast reset() via git operations (no server restart)
|
| 27 |
+
- Isolated workspace per environment instance
|
| 28 |
+
- Shared repository cache across tasks
|
| 29 |
+
- Reproducible base states from specific commits
|
| 30 |
+
|
| 31 |
+
Architecture:
|
| 32 |
+
Shared Gitea Service (external)
|
| 33 |
+
↓
|
| 34 |
+
GitTaskEnvironment instances (many)
|
| 35 |
+
↓
|
| 36 |
+
Isolated workspaces (/workspace)
|
| 37 |
+
|
| 38 |
+
Args:
|
| 39 |
+
gitea_url: URL of shared Gitea service (e.g., "http://gitea:3000")
|
| 40 |
+
username: Gitea username for authentication
|
| 41 |
+
password: Gitea password for authentication
|
| 42 |
+
workspace_dir: Directory for git operations (default: /workspace)
|
| 43 |
+
task_repos: Dict mapping task names to (repo_name, commit) tuples
|
| 44 |
+
for pre-configuring task base states
|
| 45 |
+
|
| 46 |
+
Example (Basic):
|
| 47 |
+
>>> env = GitTaskEnvironment(gitea_url="http://localhost:3000")
|
| 48 |
+
>>> obs = env.reset()
|
| 49 |
+
>>> # Clone and work
|
| 50 |
+
>>> from ..models import GitAction
|
| 51 |
+
>>> obs = env.step(GitAction(action_type="clone_repo", repo_name="my-repo"))
|
| 52 |
+
>>> obs = env.step(GitAction(action_type="execute_git_command", command="status", working_dir="my-repo"))
|
| 53 |
+
|
| 54 |
+
Example (Task-based):
|
| 55 |
+
>>> # Pre-configure tasks with specific repo states
|
| 56 |
+
>>> env = GitTaskEnvironment(
|
| 57 |
+
... gitea_url="http://localhost:3000",
|
| 58 |
+
... task_repos={
|
| 59 |
+
... "task1": ("my-repo", "abc123"), # Specific commit
|
| 60 |
+
... "task2": ("my-repo", "def456"), # Different commit
|
| 61 |
+
... }
|
| 62 |
+
... )
|
| 63 |
+
>>> # Reset to task1 base state
|
| 64 |
+
>>> obs = env.reset(task_id="task1") # Fast! Just git reset
|
| 65 |
+
>>> # Work on task...
|
| 66 |
+
>>> # Reset to task2 base state
|
| 67 |
+
>>> obs = env.reset(task_id="task2") # Fast reset to different state
|
| 68 |
+
"""
|
| 69 |
+
|
| 70 |
+
def __init__(
|
| 71 |
+
self,
|
| 72 |
+
gitea_url: str,
|
| 73 |
+
username: str,
|
| 74 |
+
password: str,
|
| 75 |
+
workspace_dir: str = "/workspace",
|
| 76 |
+
task_repos: dict[str, tuple[str, str]] | None = None,
|
| 77 |
+
):
|
| 78 |
+
"""Initialize Git Task Environment."""
|
| 79 |
+
super().__init__()
|
| 80 |
+
self.workspace_dir = workspace_dir
|
| 81 |
+
self.task_repos = task_repos or {}
|
| 82 |
+
|
| 83 |
+
# Initialize Git server client (connects to external Gitea)
|
| 84 |
+
self._git_client = GitServerClient(
|
| 85 |
+
gitea_url=gitea_url,
|
| 86 |
+
username=username,
|
| 87 |
+
password=password,
|
| 88 |
+
workspace_dir=workspace_dir,
|
| 89 |
+
)
|
| 90 |
+
|
| 91 |
+
# Initialize state
|
| 92 |
+
self._state = GitState(workspace_path=workspace_dir)
|
| 93 |
+
self._current_task_id: str | None = None
|
| 94 |
+
|
| 95 |
+
# Wait for Gitea to be ready
|
| 96 |
+
if self._git_client.wait_for_ready():
|
| 97 |
+
self._state.gitea_ready = True
|
| 98 |
+
else:
|
| 99 |
+
print("Warning: Gitea server not ready")
|
| 100 |
+
self._state.gitea_ready = False
|
| 101 |
+
|
| 102 |
+
def reset(self, task_id: str | None = None) -> Observation:
|
| 103 |
+
"""
|
| 104 |
+
Reset environment to clean state.
|
| 105 |
+
|
| 106 |
+
This is optimized for task-based workflows:
|
| 107 |
+
- If task_id specified and configured: fast reset to that task's base state
|
| 108 |
+
- If workspace exists: git reset --hard (very fast, <1s)
|
| 109 |
+
- Otherwise: clone from Gitea (slower, ~5-10s)
|
| 110 |
+
|
| 111 |
+
Args:
|
| 112 |
+
task_id: Optional task identifier for task-specific base states
|
| 113 |
+
|
| 114 |
+
Returns:
|
| 115 |
+
Initial observation indicating environment is ready
|
| 116 |
+
"""
|
| 117 |
+
# Initialize fresh state
|
| 118 |
+
self._state = GitState(
|
| 119 |
+
episode_id=str(uuid.uuid4()),
|
| 120 |
+
step_count=0,
|
| 121 |
+
gitea_ready=self._git_client.is_ready,
|
| 122 |
+
workspace_path=self.workspace_dir,
|
| 123 |
+
)
|
| 124 |
+
|
| 125 |
+
self._current_task_id = task_id
|
| 126 |
+
|
| 127 |
+
# If task_id provided and configured, set up task base state
|
| 128 |
+
if task_id and task_id in self.task_repos:
|
| 129 |
+
repo_name, commit = self.task_repos[task_id]
|
| 130 |
+
|
| 131 |
+
try:
|
| 132 |
+
if self._git_client.workspace_exists(repo_name):
|
| 133 |
+
# Fast path: workspace exists, just reset
|
| 134 |
+
self._git_client.reset_workspace(repo_name, commit)
|
| 135 |
+
message = f"Reset to task '{task_id}' base state (repo: {repo_name}@{commit})"
|
| 136 |
+
else:
|
| 137 |
+
# Slower path: clone fresh
|
| 138 |
+
self._git_client.clone_to_workspace(repo_name, commit=commit)
|
| 139 |
+
message = f"Initialized task '{task_id}' (repo: {repo_name}@{commit})"
|
| 140 |
+
|
| 141 |
+
current_commit = self._git_client.get_current_commit(repo_name)
|
| 142 |
+
|
| 143 |
+
return GitObservation(
|
| 144 |
+
success=True,
|
| 145 |
+
message=message,
|
| 146 |
+
output=f"Workspace: {self.workspace_dir}/{repo_name}\nCommit: {current_commit}\nTask: {task_id}",
|
| 147 |
+
)
|
| 148 |
+
except Exception as e:
|
| 149 |
+
return GitObservation(
|
| 150 |
+
success=False,
|
| 151 |
+
message=f"Failed to reset task '{task_id}'",
|
| 152 |
+
error=str(e),
|
| 153 |
+
)
|
| 154 |
+
|
| 155 |
+
# Default reset: just ready state, no pre-configured repos
|
| 156 |
+
return GitObservation(
|
| 157 |
+
success=True,
|
| 158 |
+
message="Git task environment ready.",
|
| 159 |
+
output=f"Workspace: {self.workspace_dir}\nGitea: {self._git_client.gitea_url}\nUse GitAction with action_type='clone_repo' to clone repositories.",
|
| 160 |
+
)
|
| 161 |
+
|
| 162 |
+
def step(self, action: Action) -> Observation:
|
| 163 |
+
"""
|
| 164 |
+
Execute a Git action and return observation.
|
| 165 |
+
|
| 166 |
+
Supported action types:
|
| 167 |
+
- "clone_repo": Clone repository to workspace
|
| 168 |
+
- "execute_git_command": Execute git command
|
| 169 |
+
- "list_repos": List available repositories
|
| 170 |
+
|
| 171 |
+
Args:
|
| 172 |
+
action: GitAction to execute
|
| 173 |
+
|
| 174 |
+
Returns:
|
| 175 |
+
GitObservation with execution results
|
| 176 |
+
"""
|
| 177 |
+
if not isinstance(action, GitAction):
|
| 178 |
+
raise ValueError(f"Expected GitAction, got {type(action)}")
|
| 179 |
+
|
| 180 |
+
# Update step count
|
| 181 |
+
self._state.step_count += 1
|
| 182 |
+
|
| 183 |
+
# Route to appropriate handler based on action_type
|
| 184 |
+
try:
|
| 185 |
+
if action.action_type == "clone_repo":
|
| 186 |
+
return self._handle_clone_repo(action)
|
| 187 |
+
elif action.action_type == "list_repos":
|
| 188 |
+
return self._handle_list_repos(action)
|
| 189 |
+
elif action.action_type == "execute_git_command":
|
| 190 |
+
return self._handle_git_command(action)
|
| 191 |
+
else:
|
| 192 |
+
return GitObservation(
|
| 193 |
+
success=False,
|
| 194 |
+
message=f"Action not supported in task mode: {type(action).__name__}",
|
| 195 |
+
error="Use shared Gitea for repository migration/creation",
|
| 196 |
+
)
|
| 197 |
+
except Exception as e:
|
| 198 |
+
return GitObservation(
|
| 199 |
+
success=False, message=f"Action failed: {str(e)}", error=str(e)
|
| 200 |
+
)
|
| 201 |
+
|
| 202 |
+
def _handle_clone_repo(self, action: GitAction) -> GitObservation:
|
| 203 |
+
"""Handle repository clone action."""
|
| 204 |
+
try:
|
| 205 |
+
# Determine commit to use
|
| 206 |
+
commit = "main" # Default
|
| 207 |
+
|
| 208 |
+
# If this repo is part of current task config, use that commit
|
| 209 |
+
if (
|
| 210 |
+
self._current_task_id
|
| 211 |
+
and self._current_task_id in self.task_repos
|
| 212 |
+
):
|
| 213 |
+
task_repo, task_commit = self.task_repos[self._current_task_id]
|
| 214 |
+
if task_repo == action.repo_name:
|
| 215 |
+
commit = task_commit
|
| 216 |
+
|
| 217 |
+
clone_path = self._git_client.clone_to_workspace(
|
| 218 |
+
action.repo_name, action.target_dir, commit=commit
|
| 219 |
+
)
|
| 220 |
+
|
| 221 |
+
return GitObservation(
|
| 222 |
+
success=True,
|
| 223 |
+
message=f"Successfully cloned {action.repo_name}",
|
| 224 |
+
output=f"Cloned to: {clone_path}\nCommit: {commit}",
|
| 225 |
+
)
|
| 226 |
+
except Exception as e:
|
| 227 |
+
return GitObservation(
|
| 228 |
+
success=False,
|
| 229 |
+
message=f"Failed to clone repository: {action.repo_name}",
|
| 230 |
+
error=str(e),
|
| 231 |
+
)
|
| 232 |
+
|
| 233 |
+
def _handle_list_repos(self, action: GitAction) -> GitObservation:
|
| 234 |
+
"""Handle list repositories action."""
|
| 235 |
+
try:
|
| 236 |
+
repos = self._git_client.list_repositories()
|
| 237 |
+
|
| 238 |
+
# Format output
|
| 239 |
+
if not repos:
|
| 240 |
+
output = "No repositories available."
|
| 241 |
+
else:
|
| 242 |
+
output = "Available repositories:\n"
|
| 243 |
+
for repo in repos:
|
| 244 |
+
output += f" - {repo['name']}: {repo['clone_url']}\n"
|
| 245 |
+
if repo.get("description"):
|
| 246 |
+
output += f" {repo['description']}\n"
|
| 247 |
+
|
| 248 |
+
return GitObservation(
|
| 249 |
+
success=True,
|
| 250 |
+
message=f"Found {len(repos)} repositories",
|
| 251 |
+
output=output,
|
| 252 |
+
repos=repos,
|
| 253 |
+
)
|
| 254 |
+
except Exception as e:
|
| 255 |
+
return GitObservation(
|
| 256 |
+
success=False, message="Failed to list repositories", error=str(e)
|
| 257 |
+
)
|
| 258 |
+
|
| 259 |
+
def _handle_git_command(self, action: GitAction) -> GitObservation:
|
| 260 |
+
"""Handle git command execution action."""
|
| 261 |
+
try:
|
| 262 |
+
exit_code, stdout, stderr = self._git_client.execute_git_command(
|
| 263 |
+
action.command, action.working_dir
|
| 264 |
+
)
|
| 265 |
+
|
| 266 |
+
success = exit_code == 0
|
| 267 |
+
message = f"Git command {'succeeded' if success else 'failed'}"
|
| 268 |
+
|
| 269 |
+
return GitObservation(
|
| 270 |
+
success=success, message=message, output=stdout, error=stderr
|
| 271 |
+
)
|
| 272 |
+
except Exception as e:
|
| 273 |
+
return GitObservation(
|
| 274 |
+
success=False,
|
| 275 |
+
message=f"Failed to execute git command: {action.command}",
|
| 276 |
+
error=str(e),
|
| 277 |
+
)
|
| 278 |
+
|
| 279 |
+
@property
|
| 280 |
+
def state(self) -> GitState:
|
| 281 |
+
"""Get current environment state."""
|
| 282 |
+
return self._state
|
models.py
ADDED
|
@@ -0,0 +1,72 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
|
| 3 |
+
"""
|
| 4 |
+
envs/git_env/models.py
|
| 5 |
+
--------------------------------
|
| 6 |
+
Action/Observation types for the Git environment with Gitea server.
|
| 7 |
+
"""
|
| 8 |
+
|
| 9 |
+
from __future__ import annotations
|
| 10 |
+
|
| 11 |
+
from pydantic import Field
|
| 12 |
+
from typing import Optional
|
| 13 |
+
|
| 14 |
+
from openenv.core.env_server import Action, Observation, State
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
class GitAction(Action):
|
| 18 |
+
"""
|
| 19 |
+
Action for Git environment operations.
|
| 20 |
+
|
| 21 |
+
This unified action class supports multiple operation types:
|
| 22 |
+
- clone_repo: Clone a repository from Gitea to workspace
|
| 23 |
+
- list_repos: List all available repositories
|
| 24 |
+
- execute_git_command: Execute a git command in workspace
|
| 25 |
+
|
| 26 |
+
Attributes:
|
| 27 |
+
action_type: Type of operation ("clone_repo", "list_repos", "execute_git_command")
|
| 28 |
+
repo_name: Name of repository (for clone_repo, execute_git_command)
|
| 29 |
+
target_dir: Target directory for clone (optional)
|
| 30 |
+
command: Git command to execute (for execute_git_command)
|
| 31 |
+
working_dir: Working directory relative to workspace (for execute_git_command)
|
| 32 |
+
"""
|
| 33 |
+
|
| 34 |
+
action_type: str = "list_repos"
|
| 35 |
+
repo_name: str = ""
|
| 36 |
+
target_dir: Optional[str] = None
|
| 37 |
+
command: str = ""
|
| 38 |
+
working_dir: str = ""
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
class GitObservation(Observation):
|
| 42 |
+
"""
|
| 43 |
+
Result of executing a Git action.
|
| 44 |
+
|
| 45 |
+
Attributes:
|
| 46 |
+
success: Whether the action was successful
|
| 47 |
+
message: Human-readable message about the result
|
| 48 |
+
output: Command output or detailed result
|
| 49 |
+
error: Error message if action failed
|
| 50 |
+
repos: List of repositories (for list_repos action)
|
| 51 |
+
"""
|
| 52 |
+
|
| 53 |
+
success: bool = False
|
| 54 |
+
message: str = ""
|
| 55 |
+
output: str = ""
|
| 56 |
+
error: str = ""
|
| 57 |
+
repos: list[dict[str, str]] = Field(default_factory=list)
|
| 58 |
+
|
| 59 |
+
|
| 60 |
+
class GitState(State):
|
| 61 |
+
"""
|
| 62 |
+
State for Git environment.
|
| 63 |
+
|
| 64 |
+
Attributes:
|
| 65 |
+
episode_id: Unique identifier for the episode
|
| 66 |
+
step_count: Number of steps taken
|
| 67 |
+
gitea_ready: Whether Gitea server is accessible
|
| 68 |
+
workspace_path: Path to the workspace directory
|
| 69 |
+
"""
|
| 70 |
+
|
| 71 |
+
gitea_ready: bool = False
|
| 72 |
+
workspace_path: str = "/workspace"
|
pyproject.toml
ADDED
|
@@ -0,0 +1,114 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[build-system]
|
| 2 |
+
requires = ["setuptools>=45", "wheel"]
|
| 3 |
+
build-backend = "setuptools.build_meta"
|
| 4 |
+
|
| 5 |
+
[project]
|
| 6 |
+
name = "openenv-core"
|
| 7 |
+
version = "0.2.0"
|
| 8 |
+
description = "A unified framework for reinforcement learning environments"
|
| 9 |
+
readme = "README.md"
|
| 10 |
+
requires-python = ">=3.10"
|
| 11 |
+
dependencies = [
|
| 12 |
+
# Core shared dependencies - minimal set required for all environments
|
| 13 |
+
# Heavy dependencies (torch, numpy, smolagents, etc.) should be in
|
| 14 |
+
# individual environment pyproject.toml files
|
| 15 |
+
"fastapi>=0.104.0",
|
| 16 |
+
"pydantic>=2.0.0",
|
| 17 |
+
"uvicorn>=0.24.0",
|
| 18 |
+
"requests>=2.25.0",
|
| 19 |
+
# CLI dependencies
|
| 20 |
+
"typer>=0.9.0",
|
| 21 |
+
"rich>=13.0.0",
|
| 22 |
+
"pyyaml>=6.0",
|
| 23 |
+
"huggingface_hub>=0.20.0",
|
| 24 |
+
"openai>=2.7.2",
|
| 25 |
+
"tomli>=2.3.0",
|
| 26 |
+
"tomli-w>=1.2.0",
|
| 27 |
+
"websockets>=15.0.1",
|
| 28 |
+
# MCP support
|
| 29 |
+
"fastmcp>=2.0.0",
|
| 30 |
+
# Web UI dependencies
|
| 31 |
+
"gradio>=4.0.0",
|
| 32 |
+
]
|
| 33 |
+
|
| 34 |
+
[project.optional-dependencies]
|
| 35 |
+
core = [
|
| 36 |
+
"fastapi>=0.104.0",
|
| 37 |
+
"pydantic>=2.0.0",
|
| 38 |
+
"uvicorn>=0.24.0",
|
| 39 |
+
"requests>=2.25.0",
|
| 40 |
+
"websockets>=15.0.1",
|
| 41 |
+
]
|
| 42 |
+
cli = [
|
| 43 |
+
"typer>=0.9.0",
|
| 44 |
+
"rich>=13.0.0",
|
| 45 |
+
"pyyaml>=6.0",
|
| 46 |
+
"huggingface_hub>=0.20.0",
|
| 47 |
+
"openai>=2.7.2",
|
| 48 |
+
"tomli>=2.3.0",
|
| 49 |
+
"tomli-w>=1.2.0",
|
| 50 |
+
]
|
| 51 |
+
all = [
|
| 52 |
+
"openenv-core[core]",
|
| 53 |
+
"openenv-core[cli]",
|
| 54 |
+
]
|
| 55 |
+
daytona = [
|
| 56 |
+
"daytona>=0.136.0",
|
| 57 |
+
"pyyaml>=6.0",
|
| 58 |
+
]
|
| 59 |
+
|
| 60 |
+
[project.scripts]
|
| 61 |
+
openenv = "openenv.cli.__main__:main"
|
| 62 |
+
|
| 63 |
+
[tool.setuptools]
|
| 64 |
+
package-dir = {"" = "src"}
|
| 65 |
+
include-package-data = true
|
| 66 |
+
|
| 67 |
+
[tool.setuptools.package-data]
|
| 68 |
+
"openenv.cli" = ["templates/**/*"]
|
| 69 |
+
|
| 70 |
+
[tool.setuptools.packages.find]
|
| 71 |
+
where = ["src"]
|
| 72 |
+
|
| 73 |
+
[tool.coverage.run]
|
| 74 |
+
omit = [
|
| 75 |
+
"openenv/cli/templates/**",
|
| 76 |
+
"**/templates/**",
|
| 77 |
+
"openenv/cli/__main__.py",
|
| 78 |
+
]
|
| 79 |
+
|
| 80 |
+
[tool.coverage.report]
|
| 81 |
+
exclude_lines = [
|
| 82 |
+
"pragma: no cover",
|
| 83 |
+
"def __repr__",
|
| 84 |
+
"raise AssertionError",
|
| 85 |
+
"raise NotImplementedError",
|
| 86 |
+
"if __name__ == .__main__.:",
|
| 87 |
+
"if TYPE_CHECKING:",
|
| 88 |
+
]
|
| 89 |
+
|
| 90 |
+
[tool.pytest.ini_options]
|
| 91 |
+
asyncio_mode = "auto"
|
| 92 |
+
asyncio_default_fixture_loop_scope = "function"
|
| 93 |
+
markers = [
|
| 94 |
+
"docker: Tests that require Docker to be running",
|
| 95 |
+
"network: Tests that require network access (HuggingFace, etc.)",
|
| 96 |
+
"integration: Integration tests with external resources",
|
| 97 |
+
]
|
| 98 |
+
|
| 99 |
+
[tool.ruff]
|
| 100 |
+
line-length = 88
|
| 101 |
+
|
| 102 |
+
[tool.ruff.lint]
|
| 103 |
+
select = ["E", "F", "W"]
|
| 104 |
+
ignore = [
|
| 105 |
+
"E402", # Module level import not at top of file (needed for pytest.importorskip patterns)
|
| 106 |
+
"E501", # Line too long (not enforced previously, would require large refactor)
|
| 107 |
+
]
|
| 108 |
+
|
| 109 |
+
[tool.ruff.lint.per-file-ignores]
|
| 110 |
+
# Context manager variables that are intentionally unused
|
| 111 |
+
"tests/envs/test_websockets.py" = ["F841"]
|
| 112 |
+
"tests/test_cli/test_push.py" = ["F841"]
|
| 113 |
+
# Compatibility shim module
|
| 114 |
+
"src/openenv_core/__init__.py" = ["F401"]
|
server/Dockerfile
ADDED
|
@@ -0,0 +1,33 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Dockerfile for Git Environment
|
| 2 |
+
# Connects to an external shared Gitea service for task-based isolation
|
| 3 |
+
# Optimized for fast resets and minimal resource usage
|
| 4 |
+
|
| 5 |
+
# Use the standard openenv base image
|
| 6 |
+
ARG BASE_IMAGE=openenv-base:latest
|
| 7 |
+
FROM ${BASE_IMAGE}
|
| 8 |
+
|
| 9 |
+
# Install git and curl (no Gitea binary needed - connects to external service)
|
| 10 |
+
RUN apt-get update && apt-get install -y \
|
| 11 |
+
git \
|
| 12 |
+
curl \
|
| 13 |
+
ca-certificates \
|
| 14 |
+
&& rm -rf /var/lib/apt/lists/*
|
| 15 |
+
|
| 16 |
+
# Create workspace directory for git operations
|
| 17 |
+
RUN mkdir -p /workspace && chmod 777 /workspace
|
| 18 |
+
|
| 19 |
+
# Copy core and environment code
|
| 20 |
+
COPY src/core/ /app/src/core/
|
| 21 |
+
COPY envs/git_env/ /app/envs/git_env/
|
| 22 |
+
|
| 23 |
+
# Environment variables for Gitea connection
|
| 24 |
+
# These MUST be provided at runtime via -e flags or --env-file
|
| 25 |
+
# See .env.example for required variables
|
| 26 |
+
ENV WORKSPACE_DIR=/workspace
|
| 27 |
+
|
| 28 |
+
# Health check
|
| 29 |
+
HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \
|
| 30 |
+
CMD curl -f http://localhost:8000/health || exit 1
|
| 31 |
+
|
| 32 |
+
# Run the FastAPI server
|
| 33 |
+
CMD ["uvicorn", "envs.git_env.server.app:app", "--host", "0.0.0.0", "--port", "8000"]
|
server/__init__.py
ADDED
|
File without changes
|
server/app.py
ADDED
|
@@ -0,0 +1,67 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
|
| 3 |
+
"""
|
| 4 |
+
FastAPI application for Git Environment.
|
| 5 |
+
|
| 6 |
+
This module creates an HTTP server for the Git environment that connects
|
| 7 |
+
to a shared external Gitea service for fast, isolated task resets.
|
| 8 |
+
|
| 9 |
+
Environment variables (required):
|
| 10 |
+
GITEA_URL: URL of shared Gitea service
|
| 11 |
+
GITEA_USERNAME: Gitea username
|
| 12 |
+
GITEA_PASSWORD: Gitea password
|
| 13 |
+
WORKSPACE_DIR: Workspace directory (optional, default: /workspace)
|
| 14 |
+
|
| 15 |
+
Usage:
|
| 16 |
+
# Development (with auto-reload):
|
| 17 |
+
uvicorn envs.git_env.server.app:app --reload --host 0.0.0.0 --port 8000
|
| 18 |
+
|
| 19 |
+
# Production:
|
| 20 |
+
uvicorn envs.git_env.server.app:app --host 0.0.0.0 --port 8000 --workers 4
|
| 21 |
+
|
| 22 |
+
# With custom Gitea:
|
| 23 |
+
GITEA_URL=http://my-gitea:3000 uvicorn envs.git_env.server.app:app --host 0.0.0.0 --port 8000
|
| 24 |
+
"""
|
| 25 |
+
|
| 26 |
+
import os
|
| 27 |
+
|
| 28 |
+
from openenv.core.env_server import create_app
|
| 29 |
+
|
| 30 |
+
from ..models import GitAction, GitObservation
|
| 31 |
+
from .git_task_environment import GitTaskEnvironment
|
| 32 |
+
|
| 33 |
+
# Read configuration from environment variables
|
| 34 |
+
gitea_url = os.getenv("GITEA_URL")
|
| 35 |
+
gitea_username = os.getenv("GITEA_USERNAME")
|
| 36 |
+
gitea_password = os.getenv("GITEA_PASSWORD")
|
| 37 |
+
workspace_dir = os.getenv("WORKSPACE_DIR", "/workspace")
|
| 38 |
+
|
| 39 |
+
# Validate required environment variables
|
| 40 |
+
if not gitea_url:
|
| 41 |
+
raise RuntimeError("GITEA_URL environment variable is required")
|
| 42 |
+
if not gitea_username:
|
| 43 |
+
raise RuntimeError("GITEA_USERNAME environment variable is required")
|
| 44 |
+
if not gitea_password:
|
| 45 |
+
raise RuntimeError("GITEA_PASSWORD environment variable is required")
|
| 46 |
+
|
| 47 |
+
|
| 48 |
+
# Factory function to create GitTaskEnvironment instances
|
| 49 |
+
def create_git_environment():
|
| 50 |
+
"""Factory function that creates GitTaskEnvironment with config."""
|
| 51 |
+
return GitTaskEnvironment(
|
| 52 |
+
gitea_url=gitea_url,
|
| 53 |
+
username=gitea_username,
|
| 54 |
+
password=gitea_password,
|
| 55 |
+
workspace_dir=workspace_dir,
|
| 56 |
+
)
|
| 57 |
+
|
| 58 |
+
|
| 59 |
+
# Create the app with web interface and README integration
|
| 60 |
+
# Pass the factory function instead of an instance for WebSocket session support
|
| 61 |
+
app = create_app(create_git_environment, GitAction, GitObservation, env_name="git_env")
|
| 62 |
+
|
| 63 |
+
|
| 64 |
+
if __name__ == "__main__":
|
| 65 |
+
import uvicorn
|
| 66 |
+
|
| 67 |
+
uvicorn.run(app, host="0.0.0.0", port=8000)
|
server/git_task_environment.py
ADDED
|
@@ -0,0 +1,282 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
|
| 3 |
+
"""
|
| 4 |
+
Git Task Environment - Optimized for task-based isolation.
|
| 5 |
+
|
| 6 |
+
This module provides an optimized Git environment for scenarios where:
|
| 7 |
+
- Multiple tasks share the same base repository states
|
| 8 |
+
- Tasks need fast reset() to reproducible states
|
| 9 |
+
- Each task has an isolated workspace
|
| 10 |
+
- A shared Gitea service provides repository storage
|
| 11 |
+
"""
|
| 12 |
+
|
| 13 |
+
import uuid
|
| 14 |
+
|
| 15 |
+
from openenv.core.env_server import Action, Environment, Observation
|
| 16 |
+
from openenv.core.tools import GitServerClient
|
| 17 |
+
|
| 18 |
+
from ..models import GitAction, GitObservation, GitState
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
class GitTaskEnvironment(Environment):
|
| 22 |
+
"""
|
| 23 |
+
Git Environment optimized for task-based isolation.
|
| 24 |
+
|
| 25 |
+
This environment connects to a shared Gitea service and provides:
|
| 26 |
+
- Fast reset() via git operations (no server restart)
|
| 27 |
+
- Isolated workspace per environment instance
|
| 28 |
+
- Shared repository cache across tasks
|
| 29 |
+
- Reproducible base states from specific commits
|
| 30 |
+
|
| 31 |
+
Architecture:
|
| 32 |
+
Shared Gitea Service (external)
|
| 33 |
+
↓
|
| 34 |
+
GitTaskEnvironment instances (many)
|
| 35 |
+
↓
|
| 36 |
+
Isolated workspaces (/workspace)
|
| 37 |
+
|
| 38 |
+
Args:
|
| 39 |
+
gitea_url: URL of shared Gitea service (e.g., "http://gitea:3000")
|
| 40 |
+
username: Gitea username for authentication
|
| 41 |
+
password: Gitea password for authentication
|
| 42 |
+
workspace_dir: Directory for git operations (default: /workspace)
|
| 43 |
+
task_repos: Dict mapping task names to (repo_name, commit) tuples
|
| 44 |
+
for pre-configuring task base states
|
| 45 |
+
|
| 46 |
+
Example (Basic):
|
| 47 |
+
>>> env = GitTaskEnvironment(gitea_url="http://localhost:3000")
|
| 48 |
+
>>> obs = env.reset()
|
| 49 |
+
>>> # Clone and work
|
| 50 |
+
>>> from ..models import GitAction
|
| 51 |
+
>>> obs = env.step(GitAction(action_type="clone_repo", repo_name="my-repo"))
|
| 52 |
+
>>> obs = env.step(GitAction(action_type="execute_git_command", command="status", working_dir="my-repo"))
|
| 53 |
+
|
| 54 |
+
Example (Task-based):
|
| 55 |
+
>>> # Pre-configure tasks with specific repo states
|
| 56 |
+
>>> env = GitTaskEnvironment(
|
| 57 |
+
... gitea_url="http://localhost:3000",
|
| 58 |
+
... task_repos={
|
| 59 |
+
... "task1": ("my-repo", "abc123"), # Specific commit
|
| 60 |
+
... "task2": ("my-repo", "def456"), # Different commit
|
| 61 |
+
... }
|
| 62 |
+
... )
|
| 63 |
+
>>> # Reset to task1 base state
|
| 64 |
+
>>> obs = env.reset(task_id="task1") # Fast! Just git reset
|
| 65 |
+
>>> # Work on task...
|
| 66 |
+
>>> # Reset to task2 base state
|
| 67 |
+
>>> obs = env.reset(task_id="task2") # Fast reset to different state
|
| 68 |
+
"""
|
| 69 |
+
|
| 70 |
+
def __init__(
|
| 71 |
+
self,
|
| 72 |
+
gitea_url: str,
|
| 73 |
+
username: str,
|
| 74 |
+
password: str,
|
| 75 |
+
workspace_dir: str = "/workspace",
|
| 76 |
+
task_repos: dict[str, tuple[str, str]] | None = None,
|
| 77 |
+
):
|
| 78 |
+
"""Initialize Git Task Environment."""
|
| 79 |
+
super().__init__()
|
| 80 |
+
self.workspace_dir = workspace_dir
|
| 81 |
+
self.task_repos = task_repos or {}
|
| 82 |
+
|
| 83 |
+
# Initialize Git server client (connects to external Gitea)
|
| 84 |
+
self._git_client = GitServerClient(
|
| 85 |
+
gitea_url=gitea_url,
|
| 86 |
+
username=username,
|
| 87 |
+
password=password,
|
| 88 |
+
workspace_dir=workspace_dir,
|
| 89 |
+
)
|
| 90 |
+
|
| 91 |
+
# Initialize state
|
| 92 |
+
self._state = GitState(workspace_path=workspace_dir)
|
| 93 |
+
self._current_task_id: str | None = None
|
| 94 |
+
|
| 95 |
+
# Wait for Gitea to be ready
|
| 96 |
+
if self._git_client.wait_for_ready():
|
| 97 |
+
self._state.gitea_ready = True
|
| 98 |
+
else:
|
| 99 |
+
print("Warning: Gitea server not ready")
|
| 100 |
+
self._state.gitea_ready = False
|
| 101 |
+
|
| 102 |
+
def reset(self, task_id: str | None = None) -> Observation:
|
| 103 |
+
"""
|
| 104 |
+
Reset environment to clean state.
|
| 105 |
+
|
| 106 |
+
This is optimized for task-based workflows:
|
| 107 |
+
- If task_id specified and configured: fast reset to that task's base state
|
| 108 |
+
- If workspace exists: git reset --hard (very fast, <1s)
|
| 109 |
+
- Otherwise: clone from Gitea (slower, ~5-10s)
|
| 110 |
+
|
| 111 |
+
Args:
|
| 112 |
+
task_id: Optional task identifier for task-specific base states
|
| 113 |
+
|
| 114 |
+
Returns:
|
| 115 |
+
Initial observation indicating environment is ready
|
| 116 |
+
"""
|
| 117 |
+
# Initialize fresh state
|
| 118 |
+
self._state = GitState(
|
| 119 |
+
episode_id=str(uuid.uuid4()),
|
| 120 |
+
step_count=0,
|
| 121 |
+
gitea_ready=self._git_client.is_ready,
|
| 122 |
+
workspace_path=self.workspace_dir,
|
| 123 |
+
)
|
| 124 |
+
|
| 125 |
+
self._current_task_id = task_id
|
| 126 |
+
|
| 127 |
+
# If task_id provided and configured, set up task base state
|
| 128 |
+
if task_id and task_id in self.task_repos:
|
| 129 |
+
repo_name, commit = self.task_repos[task_id]
|
| 130 |
+
|
| 131 |
+
try:
|
| 132 |
+
if self._git_client.workspace_exists(repo_name):
|
| 133 |
+
# Fast path: workspace exists, just reset
|
| 134 |
+
self._git_client.reset_workspace(repo_name, commit)
|
| 135 |
+
message = f"Reset to task '{task_id}' base state (repo: {repo_name}@{commit})"
|
| 136 |
+
else:
|
| 137 |
+
# Slower path: clone fresh
|
| 138 |
+
self._git_client.clone_to_workspace(repo_name, commit=commit)
|
| 139 |
+
message = f"Initialized task '{task_id}' (repo: {repo_name}@{commit})"
|
| 140 |
+
|
| 141 |
+
current_commit = self._git_client.get_current_commit(repo_name)
|
| 142 |
+
|
| 143 |
+
return GitObservation(
|
| 144 |
+
success=True,
|
| 145 |
+
message=message,
|
| 146 |
+
output=f"Workspace: {self.workspace_dir}/{repo_name}\nCommit: {current_commit}\nTask: {task_id}",
|
| 147 |
+
)
|
| 148 |
+
except Exception as e:
|
| 149 |
+
return GitObservation(
|
| 150 |
+
success=False,
|
| 151 |
+
message=f"Failed to reset task '{task_id}'",
|
| 152 |
+
error=str(e),
|
| 153 |
+
)
|
| 154 |
+
|
| 155 |
+
# Default reset: just ready state, no pre-configured repos
|
| 156 |
+
return GitObservation(
|
| 157 |
+
success=True,
|
| 158 |
+
message="Git task environment ready.",
|
| 159 |
+
output=f"Workspace: {self.workspace_dir}\nGitea: {self._git_client.gitea_url}\nUse GitAction with action_type='clone_repo' to clone repositories.",
|
| 160 |
+
)
|
| 161 |
+
|
| 162 |
+
def step(self, action: Action) -> Observation:
|
| 163 |
+
"""
|
| 164 |
+
Execute a Git action and return observation.
|
| 165 |
+
|
| 166 |
+
Supported action types:
|
| 167 |
+
- "clone_repo": Clone repository to workspace
|
| 168 |
+
- "execute_git_command": Execute git command
|
| 169 |
+
- "list_repos": List available repositories
|
| 170 |
+
|
| 171 |
+
Args:
|
| 172 |
+
action: GitAction to execute
|
| 173 |
+
|
| 174 |
+
Returns:
|
| 175 |
+
GitObservation with execution results
|
| 176 |
+
"""
|
| 177 |
+
if not isinstance(action, GitAction):
|
| 178 |
+
raise ValueError(f"Expected GitAction, got {type(action)}")
|
| 179 |
+
|
| 180 |
+
# Update step count
|
| 181 |
+
self._state.step_count += 1
|
| 182 |
+
|
| 183 |
+
# Route to appropriate handler based on action_type
|
| 184 |
+
try:
|
| 185 |
+
if action.action_type == "clone_repo":
|
| 186 |
+
return self._handle_clone_repo(action)
|
| 187 |
+
elif action.action_type == "list_repos":
|
| 188 |
+
return self._handle_list_repos(action)
|
| 189 |
+
elif action.action_type == "execute_git_command":
|
| 190 |
+
return self._handle_git_command(action)
|
| 191 |
+
else:
|
| 192 |
+
return GitObservation(
|
| 193 |
+
success=False,
|
| 194 |
+
message=f"Action not supported in task mode: {type(action).__name__}",
|
| 195 |
+
error="Use shared Gitea for repository migration/creation",
|
| 196 |
+
)
|
| 197 |
+
except Exception as e:
|
| 198 |
+
return GitObservation(
|
| 199 |
+
success=False, message=f"Action failed: {str(e)}", error=str(e)
|
| 200 |
+
)
|
| 201 |
+
|
| 202 |
+
def _handle_clone_repo(self, action: GitAction) -> GitObservation:
|
| 203 |
+
"""Handle repository clone action."""
|
| 204 |
+
try:
|
| 205 |
+
# Determine commit to use
|
| 206 |
+
commit = "main" # Default
|
| 207 |
+
|
| 208 |
+
# If this repo is part of current task config, use that commit
|
| 209 |
+
if (
|
| 210 |
+
self._current_task_id
|
| 211 |
+
and self._current_task_id in self.task_repos
|
| 212 |
+
):
|
| 213 |
+
task_repo, task_commit = self.task_repos[self._current_task_id]
|
| 214 |
+
if task_repo == action.repo_name:
|
| 215 |
+
commit = task_commit
|
| 216 |
+
|
| 217 |
+
clone_path = self._git_client.clone_to_workspace(
|
| 218 |
+
action.repo_name, action.target_dir, commit=commit
|
| 219 |
+
)
|
| 220 |
+
|
| 221 |
+
return GitObservation(
|
| 222 |
+
success=True,
|
| 223 |
+
message=f"Successfully cloned {action.repo_name}",
|
| 224 |
+
output=f"Cloned to: {clone_path}\nCommit: {commit}",
|
| 225 |
+
)
|
| 226 |
+
except Exception as e:
|
| 227 |
+
return GitObservation(
|
| 228 |
+
success=False,
|
| 229 |
+
message=f"Failed to clone repository: {action.repo_name}",
|
| 230 |
+
error=str(e),
|
| 231 |
+
)
|
| 232 |
+
|
| 233 |
+
def _handle_list_repos(self, action: GitAction) -> GitObservation:
|
| 234 |
+
"""Handle list repositories action."""
|
| 235 |
+
try:
|
| 236 |
+
repos = self._git_client.list_repositories()
|
| 237 |
+
|
| 238 |
+
# Format output
|
| 239 |
+
if not repos:
|
| 240 |
+
output = "No repositories available."
|
| 241 |
+
else:
|
| 242 |
+
output = "Available repositories:\n"
|
| 243 |
+
for repo in repos:
|
| 244 |
+
output += f" - {repo['name']}: {repo['clone_url']}\n"
|
| 245 |
+
if repo.get("description"):
|
| 246 |
+
output += f" {repo['description']}\n"
|
| 247 |
+
|
| 248 |
+
return GitObservation(
|
| 249 |
+
success=True,
|
| 250 |
+
message=f"Found {len(repos)} repositories",
|
| 251 |
+
output=output,
|
| 252 |
+
repos=repos,
|
| 253 |
+
)
|
| 254 |
+
except Exception as e:
|
| 255 |
+
return GitObservation(
|
| 256 |
+
success=False, message="Failed to list repositories", error=str(e)
|
| 257 |
+
)
|
| 258 |
+
|
| 259 |
+
def _handle_git_command(self, action: GitAction) -> GitObservation:
|
| 260 |
+
"""Handle git command execution action."""
|
| 261 |
+
try:
|
| 262 |
+
exit_code, stdout, stderr = self._git_client.execute_git_command(
|
| 263 |
+
action.command, action.working_dir
|
| 264 |
+
)
|
| 265 |
+
|
| 266 |
+
success = exit_code == 0
|
| 267 |
+
message = f"Git command {'succeeded' if success else 'failed'}"
|
| 268 |
+
|
| 269 |
+
return GitObservation(
|
| 270 |
+
success=success, message=message, output=stdout, error=stderr
|
| 271 |
+
)
|
| 272 |
+
except Exception as e:
|
| 273 |
+
return GitObservation(
|
| 274 |
+
success=False,
|
| 275 |
+
message=f"Failed to execute git command: {action.command}",
|
| 276 |
+
error=str(e),
|
| 277 |
+
)
|
| 278 |
+
|
| 279 |
+
@property
|
| 280 |
+
def state(self) -> GitState:
|
| 281 |
+
"""Get current environment state."""
|
| 282 |
+
return self._state
|
src/__init__.py
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Meta Platforms, Inc. and affiliates.
|
| 2 |
+
# All rights reserved.
|
| 3 |
+
#
|
| 4 |
+
# This source code is licensed under the BSD-style license found in the
|
| 5 |
+
# LICENSE file in the root directory of this source tree.
|
| 6 |
+
|
| 7 |
+
"""EnvTorch: Standardized agentic execution environments."""
|
src/openenv.egg-info/PKG-INFO
ADDED
|
@@ -0,0 +1,337 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
Metadata-Version: 2.4
|
| 2 |
+
Name: openenv
|
| 3 |
+
Version: 0.2.0
|
| 4 |
+
Summary: A unified framework for reinforcement learning environments
|
| 5 |
+
Requires-Python: >=3.10
|
| 6 |
+
Description-Content-Type: text/markdown
|
| 7 |
+
License-File: LICENSE
|
| 8 |
+
Requires-Dist: fastapi>=0.104.0
|
| 9 |
+
Requires-Dist: pydantic>=2.0.0
|
| 10 |
+
Requires-Dist: uvicorn>=0.24.0
|
| 11 |
+
Requires-Dist: requests>=2.25.0
|
| 12 |
+
Requires-Dist: typer>=0.9.0
|
| 13 |
+
Requires-Dist: rich>=13.0.0
|
| 14 |
+
Requires-Dist: pyyaml>=6.0
|
| 15 |
+
Requires-Dist: huggingface_hub>=0.20.0
|
| 16 |
+
Requires-Dist: openai>=2.7.2
|
| 17 |
+
Requires-Dist: tomli>=2.3.0
|
| 18 |
+
Requires-Dist: tomli-w>=1.2.0
|
| 19 |
+
Requires-Dist: websockets>=15.0.1
|
| 20 |
+
Provides-Extra: core
|
| 21 |
+
Requires-Dist: fastapi>=0.104.0; extra == "core"
|
| 22 |
+
Requires-Dist: pydantic>=2.0.0; extra == "core"
|
| 23 |
+
Requires-Dist: uvicorn>=0.24.0; extra == "core"
|
| 24 |
+
Requires-Dist: requests>=2.25.0; extra == "core"
|
| 25 |
+
Requires-Dist: websockets>=15.0.1; extra == "core"
|
| 26 |
+
Provides-Extra: cli
|
| 27 |
+
Requires-Dist: typer>=0.9.0; extra == "cli"
|
| 28 |
+
Requires-Dist: rich>=13.0.0; extra == "cli"
|
| 29 |
+
Requires-Dist: pyyaml>=6.0; extra == "cli"
|
| 30 |
+
Requires-Dist: huggingface_hub>=0.20.0; extra == "cli"
|
| 31 |
+
Requires-Dist: openai>=2.7.2; extra == "cli"
|
| 32 |
+
Requires-Dist: tomli>=2.3.0; extra == "cli"
|
| 33 |
+
Requires-Dist: tomli-w>=1.2.0; extra == "cli"
|
| 34 |
+
Provides-Extra: all
|
| 35 |
+
Requires-Dist: openenv[core]; extra == "all"
|
| 36 |
+
Requires-Dist: openenv[cli]; extra == "all"
|
| 37 |
+
Dynamic: license-file
|
| 38 |
+
|
| 39 |
+
# <img width="35" height="35" alt="image" src="https://github.com/user-attachments/assets/2700a971-e5d6-4036-b03f-2f89c9791609" /> OpenEnv: Agentic Execution Environments
|
| 40 |
+
|
| 41 |
+
An e2e framework for creating, deploying and using isolated execution environments for agentic RL training, built using Gymnasium style simple APIs.
|
| 42 |
+
|
| 43 |
+
[](https://pypi.org/project/openenv/)
|
| 44 |
+
[](https://discord.gg/YsTYBh6PD9)
|
| 45 |
+
[](https://colab.research.google.com/github/meta-pytorch/OpenEnv/blob/main/examples/OpenEnv_Tutorial.ipynb)
|
| 46 |
+
[](https://meta-pytorch.org/OpenEnv/)
|
| 47 |
+
|
| 48 |
+
---
|
| 49 |
+
|
| 50 |
+
**🚀 Featured Example:** Train LLMs to play BlackJack using [torchforge](https://github.com/meta-pytorch/torchforge) (PyTorch's agentic RL framework): [`examples/grpo_blackjack/`](examples/grpo_blackjack/)
|
| 51 |
+
|
| 52 |
+
## OpenEnv on partner platforms:
|
| 53 |
+
|
| 54 |
+
- [Lightning AI Studio](https://lightning.ai/environments?section=featured)
|
| 55 |
+
- [TRL example](https://huggingface.co/docs/trl/main/en/openenv)
|
| 56 |
+
- [Unsloth Google Colab](https://colab.research.google.com/github/unslothai/notebooks/blob/main/nb/OpenEnv_gpt_oss_(20B)_Reinforcement_Learning_2048_Game.ipynb)
|
| 57 |
+
- [ART example](https://art.openpipe.ai/integrations/openenv-integration)
|
| 58 |
+
- [Oumi example](https://github.com/oumi-ai/oumi/blob/main/notebooks/Oumi%20-%20OpenEnv%20GRPO%20with%20trl.ipynb)
|
| 59 |
+
|
| 60 |
+
## Overview
|
| 61 |
+
|
| 62 |
+
OpenEnv provides a standard for interacting with agentic execution environments via simple Gymnasium style APIs - `step()`, `reset()`, `state()`. Users of agentic execution environments can interact with the environment during RL training loops using these simple APIs.
|
| 63 |
+
|
| 64 |
+
In addition to making it easier for researchers and RL framework writers, we also provide tools for environment creators making it easier for them to create richer environments and make them available over familiar protocols like HTTP and packaged using canonical technologies like docker. Environment creators can use the OpenEnv framework to create environments that are isolated, secure, and easy to deploy and use.
|
| 65 |
+
|
| 66 |
+
The OpenEnv CLI (`openenv`) provides commands to initialize new environments and deploy them to Hugging Face Spaces.
|
| 67 |
+
|
| 68 |
+
> ⚠️ **Early Development Warning** OpenEnv is currently in an experimental
|
| 69 |
+
> stage. You should expect bugs, incomplete features, and APIs that may change
|
| 70 |
+
> in future versions. The project welcomes bugfixes, but to make sure things are
|
| 71 |
+
> well coordinated you should discuss any significant change before starting the
|
| 72 |
+
> work. It's recommended that you signal your intention to contribute in the
|
| 73 |
+
> issue tracker, either by filing a new issue or by claiming an existing one.
|
| 74 |
+
|
| 75 |
+
### RFCs
|
| 76 |
+
|
| 77 |
+
Below is a list of active and historical RFCs for OpenEnv. RFCs are proposals for major changes or features. Please review and contribute!
|
| 78 |
+
|
| 79 |
+
- [RFC 001: Baseline API and Interface Specifications](https://github.com/meta-pytorch/OpenEnv/pull/26)
|
| 80 |
+
|
| 81 |
+
## Architecture
|
| 82 |
+
|
| 83 |
+
### Component Overview
|
| 84 |
+
|
| 85 |
+
```
|
| 86 |
+
┌─────────────────────────────────────────────────────────┐
|
| 87 |
+
│ Client Application │
|
| 88 |
+
│ ┌────────────────┐ ┌──────────────────┐ │
|
| 89 |
+
│ │ EchoEnv │ │ CodingEnv │ │
|
| 90 |
+
│ │ (HTTPEnvClient)│ �� (HTTPEnvClient) │ │
|
| 91 |
+
│ └────────┬───────┘ └────────┬─────────┘ │
|
| 92 |
+
└───────────┼───────────────────────────────┼─────────────┘
|
| 93 |
+
│ HTTP │ HTTP
|
| 94 |
+
│ (reset, step, state) │
|
| 95 |
+
┌───────────▼───────────────────────────────▼─────────────┐
|
| 96 |
+
│ Docker Containers (Isolated) │
|
| 97 |
+
│ ┌──────────────────────┐ ┌──────────────────────┐ │
|
| 98 |
+
│ │ FastAPI Server │ │ FastAPI Server │ │
|
| 99 |
+
│ │ EchoEnvironment │ │ PythonCodeActEnv │ │
|
| 100 |
+
│ │ (Environment base) │ │ (Environment base) │ │
|
| 101 |
+
│ └──────────────────────┘ └──────────────────────┘ │
|
| 102 |
+
└─────────────────────────────────────────────────────────┘
|
| 103 |
+
```
|
| 104 |
+
|
| 105 |
+
### Core Components
|
| 106 |
+
|
| 107 |
+
#### 1. Web Interface
|
| 108 |
+
|
| 109 |
+
OpenEnv includes a built-in web interface for interactive environment exploration and debugging. The web interface provides:
|
| 110 |
+
|
| 111 |
+
- **Two-Pane Layout**: HumanAgent interaction on the left, state observation on the right
|
| 112 |
+
- **Real-time Updates**: WebSocket-based live updates without page refresh
|
| 113 |
+
- **Dynamic Forms**: Automatically generated action forms based on environment Action types
|
| 114 |
+
- **Action History**: Complete log of all actions taken and their results
|
| 115 |
+
|
| 116 |
+
The web interface is **conditionally enabled** based on environment variables:
|
| 117 |
+
|
| 118 |
+
- **Local Development**: Disabled by default for lightweight development
|
| 119 |
+
- **Manual Override**: Enable with `ENABLE_WEB_INTERFACE=true`
|
| 120 |
+
|
| 121 |
+
To use the web interface:
|
| 122 |
+
|
| 123 |
+
```python
|
| 124 |
+
from openenv.core.env_server import create_web_interface_app
|
| 125 |
+
from your_env.models import YourAction, YourObservation
|
| 126 |
+
from your_env.server.your_environment import YourEnvironment
|
| 127 |
+
|
| 128 |
+
env = YourEnvironment()
|
| 129 |
+
app = create_web_interface_app(env, YourAction, YourObservation)
|
| 130 |
+
```
|
| 131 |
+
|
| 132 |
+
When enabled, open `http://localhost:8000/web` in your browser to interact with the environment.
|
| 133 |
+
|
| 134 |
+
#### 2. Environment (Server-Side)
|
| 135 |
+
Base class for implementing environment logic:
|
| 136 |
+
- **`reset()`**: Initialize a new episode, returns initial `Observation`
|
| 137 |
+
- **`step(action)`**: Execute an `Action`, returns resulting `Observation`
|
| 138 |
+
- **`state()`**: Access episode metadata (`State` with episode_id, step_count, etc.)
|
| 139 |
+
|
| 140 |
+
#### 3. HTTPEnvClient (Client-Side)
|
| 141 |
+
Base class for HTTP communication:
|
| 142 |
+
- Handles HTTP requests to environment server
|
| 143 |
+
- Contains a utility to spin up a docker container locally for the corresponding environment
|
| 144 |
+
- Type-safe action/observation parsing
|
| 145 |
+
|
| 146 |
+
#### 4. Container Providers
|
| 147 |
+
Manage container deployment:
|
| 148 |
+
- `LocalDockerProvider`: Run containers on local Docker daemon
|
| 149 |
+
- `KubernetesProvider`: Deploy to K8s clusters (future)
|
| 150 |
+
|
| 151 |
+
#### 5. Models
|
| 152 |
+
Type-safe data structures:
|
| 153 |
+
- `Action`: Base class for environment actions
|
| 154 |
+
- `Observation`: Base class for environment observations
|
| 155 |
+
- `State`: Episode state tracking
|
| 156 |
+
- `StepResult`: Combines observation, reward, done flag
|
| 157 |
+
|
| 158 |
+
## Project Structure
|
| 159 |
+
|
| 160 |
+
### For Environment Creators
|
| 161 |
+
|
| 162 |
+
Use the CLI to quickly scaffold a new environment:
|
| 163 |
+
|
| 164 |
+
```bash
|
| 165 |
+
openenv init my_env
|
| 166 |
+
```
|
| 167 |
+
|
| 168 |
+
This creates the following structure:
|
| 169 |
+
|
| 170 |
+
```
|
| 171 |
+
my_env/
|
| 172 |
+
├── .dockerignore # Docker build exclusions
|
| 173 |
+
├── __init__.py # Export YourAction, YourObservation, YourEnv
|
| 174 |
+
├── models.py # Define Action, Observation, State dataclasses
|
| 175 |
+
├── client.py # Implement YourEnv(HTTPEnvClient)
|
| 176 |
+
├── README.md # Document your environment
|
| 177 |
+
├── openenv.yaml # Environment manifest
|
| 178 |
+
├── pyproject.toml # Dependencies and package configuration
|
| 179 |
+
├── outputs/ # Runtime outputs (logs, evals) - gitignored
|
| 180 |
+
│ ├── logs/
|
| 181 |
+
│ └── evals/
|
| 182 |
+
└── server/
|
| 183 |
+
├── your_environment.py # Implement YourEnvironment(Environment)
|
| 184 |
+
├── app.py # Create FastAPI app
|
| 185 |
+
├── requirements.txt # Dependencies for Docker (can be generated)
|
| 186 |
+
└── Dockerfile # Define container image
|
| 187 |
+
```
|
| 188 |
+
|
| 189 |
+
#### Dependency Management
|
| 190 |
+
|
| 191 |
+
OpenEnv uses `pyproject.toml` as the primary dependency specification:
|
| 192 |
+
|
| 193 |
+
- **Environment-level `pyproject.toml`**: Each environment defines its own dependencies
|
| 194 |
+
- **Root-level `pyproject.toml`**: Contains shared core dependencies (fastapi, pydantic, uvicorn)
|
| 195 |
+
- **Server `requirements.txt`**: Can be auto-generated from `pyproject.toml` for Docker builds
|
| 196 |
+
|
| 197 |
+
**Development Workflow:**
|
| 198 |
+
|
| 199 |
+
```bash
|
| 200 |
+
# Install environment in editable mode
|
| 201 |
+
cd my_env
|
| 202 |
+
pip install -e .
|
| 203 |
+
|
| 204 |
+
# Or using uv (faster)
|
| 205 |
+
uv pip install -e .
|
| 206 |
+
|
| 207 |
+
# Run server locally without Docker
|
| 208 |
+
uv run server --host 0.0.0.0 --port 8000
|
| 209 |
+
```
|
| 210 |
+
|
| 211 |
+
**Benefits:**
|
| 212 |
+
- ✅ **Client-side extensions**: Modify client classes locally without repo changes
|
| 213 |
+
- ✅ **Better dependency management**: Clear separation between environments
|
| 214 |
+
- ✅ **Flexible workflows**: Use pip, uv, or Docker for different scenarios
|
| 215 |
+
- ✅ **CI/CD ready**: Automated dependency generation and validation
|
| 216 |
+
|
| 217 |
+
See [`envs/README.md`](envs/README.md) for a complete guide on building environments.
|
| 218 |
+
|
| 219 |
+
### For Environment Users
|
| 220 |
+
|
| 221 |
+
To use an environment:
|
| 222 |
+
1. Import from `envs.your_env`: `from envs.echo_env import EchoAction, EchoEnv`
|
| 223 |
+
2. Create client: `client = EchoEnv.from_docker_image("echo-env:latest")`
|
| 224 |
+
3. Interact: `client.reset()`, `client.step(action)`, `client.state()`
|
| 225 |
+
4. Cleanup: `client.close()`
|
| 226 |
+
|
| 227 |
+
See example scripts in `examples/` directory.
|
| 228 |
+
|
| 229 |
+
## CLI Commands
|
| 230 |
+
|
| 231 |
+
The OpenEnv CLI provides commands to manage environments:
|
| 232 |
+
|
| 233 |
+
- **`openenv init <env_name>`** - Initialize a new environment from template
|
| 234 |
+
- **`openenv push [--repo-id <repo>] [--private]`** - Deploy environment to Hugging Face Spaces
|
| 235 |
+
|
| 236 |
+
### Quick Start
|
| 237 |
+
|
| 238 |
+
```bash
|
| 239 |
+
# Create a new environment
|
| 240 |
+
openenv init my_game_env
|
| 241 |
+
|
| 242 |
+
# Deploy to Hugging Face (will prompt for login if needed)
|
| 243 |
+
cd my_game_env
|
| 244 |
+
openenv push
|
| 245 |
+
```
|
| 246 |
+
|
| 247 |
+
For detailed options: `openenv init --help` and `openenv push --help`.
|
| 248 |
+
|
| 249 |
+
## Design Principles
|
| 250 |
+
|
| 251 |
+
1. **Separation of Concerns**: Clear client-server boundaries
|
| 252 |
+
2. **Type Safety**: Strongly-typed actions, observations, and state
|
| 253 |
+
3. **Container Isolation**: Each environment runs in its own container
|
| 254 |
+
4. **Simple APIs**: Minimal, intuitive interfaces
|
| 255 |
+
|
| 256 |
+
## Quick Start
|
| 257 |
+
|
| 258 |
+
### Using the Echo Environment(Example)
|
| 259 |
+
|
| 260 |
+
```python
|
| 261 |
+
from envs.echo_env import EchoAction, EchoEnv
|
| 262 |
+
|
| 263 |
+
# Automatically start container and connect
|
| 264 |
+
client = EchoEnv.from_docker_image("echo-env:latest")
|
| 265 |
+
|
| 266 |
+
# Reset the environment
|
| 267 |
+
result = client.reset()
|
| 268 |
+
print(result.observation.echoed_message) # "Echo environment ready!"
|
| 269 |
+
|
| 270 |
+
# Send messages
|
| 271 |
+
result = client.step(EchoAction(message="Hello, World!"))
|
| 272 |
+
print(result.observation.echoed_message) # "Hello, World!"
|
| 273 |
+
print(result.reward) # 1.3 (based on message length)
|
| 274 |
+
|
| 275 |
+
# Cleanup
|
| 276 |
+
client.close() # Stops and removes container
|
| 277 |
+
```
|
| 278 |
+
|
| 279 |
+
## Requirements
|
| 280 |
+
|
| 281 |
+
- Python 3.11+
|
| 282 |
+
- Docker Desktop or Docker Engine
|
| 283 |
+
- FastAPI >= 0.104.0
|
| 284 |
+
- Uvicorn >= 0.24.0
|
| 285 |
+
- Requests >= 2.25.0
|
| 286 |
+
- smolagents (for coding environment)
|
| 287 |
+
|
| 288 |
+
## Supported RL Tools
|
| 289 |
+
The goal of this project is to support a broad set of open and closed tools to help standardize the agentic RL community. If you have a project that supports OpenEnv environments, please put up a PR to add your tool name along with a link to your documentation.
|
| 290 |
+
|
| 291 |
+
### torchforge
|
| 292 |
+
See GRPO BlackJack training example: [`examples/grpo_blackjack/`](examples/grpo_blackjack/)
|
| 293 |
+
|
| 294 |
+
### TRL
|
| 295 |
+
See the [TRL example](https://huggingface.co/docs/trl/main/en/openenv) on how to integrate OpenEnv environments with GRPO training.
|
| 296 |
+
|
| 297 |
+
### Unsloth
|
| 298 |
+
See the 2048 game example based on gpt-oss: [Colab notebook](https://colab.research.google.com/github/unslothai/notebooks/blob/main/nb/OpenEnv_gpt_oss_(20B)_Reinforcement_Learning_2048_Game.ipynb)
|
| 299 |
+
|
| 300 |
+
### SkyRL
|
| 301 |
+
See the [SkyRL example](https://skyrl.readthedocs.io/en/latest/examples/openenv.html) on how to train on OpenEnv environments with SkyRL.
|
| 302 |
+
|
| 303 |
+
### ART
|
| 304 |
+
See the [ART example](https://art.openpipe.ai/integrations/openenv-integration) on how OpenEnv environments can be used to train models with ART.
|
| 305 |
+
|
| 306 |
+
### Oumi
|
| 307 |
+
See the [Oumi example](https://github.com/oumi-ai/oumi/blob/main/notebooks/Oumi%20-%20OpenEnv%20GRPO%20with%20trl.ipynb) on how OpenEnv environments can be used to train models with Oumi.
|
| 308 |
+
|
| 309 |
+
## Example Environments
|
| 310 |
+
|
| 311 |
+
### Echo Environment
|
| 312 |
+
A simple environment that echoes back messages with metadata. Perfect for:
|
| 313 |
+
- Testing the HTTP server infrastructure
|
| 314 |
+
- Learning the framework basics
|
| 315 |
+
- Verifying container deployment
|
| 316 |
+
|
| 317 |
+
See: [`envs/echo_env/README.md`](envs/echo_env/README.md)
|
| 318 |
+
|
| 319 |
+
### Coding Environment
|
| 320 |
+
Executes arbitrary Python code in a sandboxed environment. Features:
|
| 321 |
+
- Safe code execution using smolagents
|
| 322 |
+
- Capture stdout, stderr, and exit codes
|
| 323 |
+
- Persistent execution context within episodes
|
| 324 |
+
- Error handling with detailed messages
|
| 325 |
+
|
| 326 |
+
See: [`envs/coding_env/README.md`](envs/coding_env/README.md)
|
| 327 |
+
|
| 328 |
+
## Community Support & Acknowledgments
|
| 329 |
+
This is an open and community-centric project. If you would like to add your name here, please put up a pull request and tag @jspisak for review. Ty!!
|
| 330 |
+
|
| 331 |
+
Supporters include: Meta-PyTorch, Hugging Face, [Patronus AI](https://patronus.ai), [Surge AI](https://surgehq.ai), [LastMile AI](https://www.lastmileai.dev), Unsloth AI, Reflection AI, vLLM, SkyRL (UC-Berkeley), LightningAI, Axolotl AI, Stanford Scaling Intelligence Lab, Mithril, [OpenMined](https://openmined.org/), [Fleet AI](https://fleetai.com), [Halluminate](https://halluminate.ai/), [Turing](https://www.turing.com/) ..
|
| 332 |
+
|
| 333 |
+
And we'd also like to acknowledge the team at Farama Foundation as the OpenEnv API was heavily inspired by the work you all have done on Gymnasium. Cheers!
|
| 334 |
+
|
| 335 |
+
## License
|
| 336 |
+
|
| 337 |
+
BSD 3-Clause License (see [LICENSE](./LICENSE) file)
|
src/openenv.egg-info/SOURCES.txt
ADDED
|
@@ -0,0 +1,142 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
LICENSE
|
| 2 |
+
README.md
|
| 3 |
+
pyproject.toml
|
| 4 |
+
envs/atari_env/__init__.py
|
| 5 |
+
envs/atari_env/client.py
|
| 6 |
+
envs/atari_env/models.py
|
| 7 |
+
envs/atari_env/server/__init__.py
|
| 8 |
+
envs/atari_env/server/app.py
|
| 9 |
+
envs/atari_env/server/atari_environment.py
|
| 10 |
+
envs/browsergym_env/__init__.py
|
| 11 |
+
envs/browsergym_env/client.py
|
| 12 |
+
envs/browsergym_env/models.py
|
| 13 |
+
envs/browsergym_env/server/__init__.py
|
| 14 |
+
envs/browsergym_env/server/app.py
|
| 15 |
+
envs/browsergym_env/server/browsergym_environment.py
|
| 16 |
+
envs/chat_env/__init__.py
|
| 17 |
+
envs/chat_env/client.py
|
| 18 |
+
envs/chat_env/models.py
|
| 19 |
+
envs/chat_env/server/__init__.py
|
| 20 |
+
envs/chat_env/server/app.py
|
| 21 |
+
envs/chat_env/server/chat_environment.py
|
| 22 |
+
envs/chat_env/server/test_chat_env.py
|
| 23 |
+
envs/coding_env/__init__.py
|
| 24 |
+
envs/coding_env/client.py
|
| 25 |
+
envs/coding_env/models.py
|
| 26 |
+
envs/coding_env/server/__init__.py
|
| 27 |
+
envs/coding_env/server/app.py
|
| 28 |
+
envs/coding_env/server/python_codeact_env.py
|
| 29 |
+
envs/coding_env/server/python_executor.py
|
| 30 |
+
envs/coding_env/server/transforms.py
|
| 31 |
+
envs/connect4_env/__init__.py
|
| 32 |
+
envs/connect4_env/client.py
|
| 33 |
+
envs/connect4_env/models.py
|
| 34 |
+
envs/connect4_env/server/__init__.py
|
| 35 |
+
envs/connect4_env/server/app.py
|
| 36 |
+
envs/connect4_env/server/connect4_environment.py
|
| 37 |
+
envs/dipg_safety_env/__init__.py
|
| 38 |
+
envs/dipg_safety_env/client.py
|
| 39 |
+
envs/dipg_safety_env/models.py
|
| 40 |
+
envs/dipg_safety_env/server/__init__.py
|
| 41 |
+
envs/dipg_safety_env/server/app.py
|
| 42 |
+
envs/dipg_safety_env/server/dipg_environment.py
|
| 43 |
+
envs/echo_env/__init__.py
|
| 44 |
+
envs/echo_env/client.py
|
| 45 |
+
envs/echo_env/models.py
|
| 46 |
+
envs/echo_env/build/lib/server/__init__.py
|
| 47 |
+
envs/echo_env/build/lib/server/app.py
|
| 48 |
+
envs/echo_env/build/lib/server/echo_environment.py
|
| 49 |
+
envs/echo_env/server/__init__.py
|
| 50 |
+
envs/echo_env/server/app.py
|
| 51 |
+
envs/echo_env/server/echo_environment.py
|
| 52 |
+
envs/finrl_env/__init__.py
|
| 53 |
+
envs/finrl_env/client.py
|
| 54 |
+
envs/finrl_env/models.py
|
| 55 |
+
envs/finrl_env/server/__init__.py
|
| 56 |
+
envs/finrl_env/server/app.py
|
| 57 |
+
envs/finrl_env/server/finrl_environment.py
|
| 58 |
+
envs/git_env/__init__.py
|
| 59 |
+
envs/git_env/client.py
|
| 60 |
+
envs/git_env/models.py
|
| 61 |
+
envs/git_env/server/__init__.py
|
| 62 |
+
envs/git_env/server/app.py
|
| 63 |
+
envs/git_env/server/git_task_environment.py
|
| 64 |
+
envs/openspiel_env/__init__.py
|
| 65 |
+
envs/openspiel_env/client.py
|
| 66 |
+
envs/openspiel_env/models.py
|
| 67 |
+
envs/openspiel_env/server/__init__.py
|
| 68 |
+
envs/openspiel_env/server/app.py
|
| 69 |
+
envs/openspiel_env/server/openspiel_environment.py
|
| 70 |
+
envs/openspiel_env/server/opponent_policies.py
|
| 71 |
+
envs/play/build/lib/server/__init__.py
|
| 72 |
+
envs/play/build/lib/server/app.py
|
| 73 |
+
envs/play/build/lib/server/play_environment.py
|
| 74 |
+
envs/sumo_rl_env/__init__.py
|
| 75 |
+
envs/sumo_rl_env/client.py
|
| 76 |
+
envs/sumo_rl_env/models.py
|
| 77 |
+
envs/sumo_rl_env/server/__init__.py
|
| 78 |
+
envs/sumo_rl_env/server/app.py
|
| 79 |
+
envs/sumo_rl_env/server/sumo_environment.py
|
| 80 |
+
envs/textarena_env/__init__.py
|
| 81 |
+
envs/textarena_env/client.py
|
| 82 |
+
envs/textarena_env/models.py
|
| 83 |
+
envs/textarena_env/rewards.py
|
| 84 |
+
envs/textarena_env/build/lib/server/__init__.py
|
| 85 |
+
envs/textarena_env/build/lib/server/app.py
|
| 86 |
+
envs/textarena_env/build/lib/server/environment.py
|
| 87 |
+
envs/textarena_env/server/__init__.py
|
| 88 |
+
envs/textarena_env/server/app.py
|
| 89 |
+
envs/textarena_env/server/environment.py
|
| 90 |
+
src/openenv/__init__.py
|
| 91 |
+
src/openenv.egg-info/PKG-INFO
|
| 92 |
+
src/openenv.egg-info/SOURCES.txt
|
| 93 |
+
src/openenv.egg-info/dependency_links.txt
|
| 94 |
+
src/openenv.egg-info/entry_points.txt
|
| 95 |
+
src/openenv.egg-info/requires.txt
|
| 96 |
+
src/openenv.egg-info/top_level.txt
|
| 97 |
+
src/openenv/cli/__init__.py
|
| 98 |
+
src/openenv/cli/__main__.py
|
| 99 |
+
src/openenv/cli/_cli_utils.py
|
| 100 |
+
src/openenv/cli/_validation.py
|
| 101 |
+
src/openenv/cli/commands/__init__.py
|
| 102 |
+
src/openenv/cli/commands/build.py
|
| 103 |
+
src/openenv/cli/commands/init.py
|
| 104 |
+
src/openenv/cli/commands/push.py
|
| 105 |
+
src/openenv/cli/commands/serve.py
|
| 106 |
+
src/openenv/cli/commands/validate.py
|
| 107 |
+
src/openenv/cli/templates/__init__.py
|
| 108 |
+
src/openenv/cli/templates/__pycache__/__init__.cpython-311.pyc
|
| 109 |
+
src/openenv/cli/templates/__pycache__/__init__.cpython-313.pyc
|
| 110 |
+
src/openenv/cli/templates/openenv_env/README.md
|
| 111 |
+
src/openenv/cli/templates/openenv_env/__init__.py
|
| 112 |
+
src/openenv/cli/templates/openenv_env/client.py
|
| 113 |
+
src/openenv/cli/templates/openenv_env/models.py
|
| 114 |
+
src/openenv/cli/templates/openenv_env/openenv.yaml
|
| 115 |
+
src/openenv/cli/templates/openenv_env/pyproject.toml
|
| 116 |
+
src/openenv/cli/templates/openenv_env/server/Dockerfile
|
| 117 |
+
src/openenv/cli/templates/openenv_env/server/__ENV_NAME___environment.py
|
| 118 |
+
src/openenv/cli/templates/openenv_env/server/__init__.py
|
| 119 |
+
src/openenv/cli/templates/openenv_env/server/app.py
|
| 120 |
+
src/openenv/cli/templates/openenv_env/server/requirements.txt
|
| 121 |
+
src/openenv/core/__init__.py
|
| 122 |
+
src/openenv/core/client_types.py
|
| 123 |
+
src/openenv/core/env_client.py
|
| 124 |
+
src/openenv/core/utils.py
|
| 125 |
+
src/openenv/core/containers/__init__.py
|
| 126 |
+
src/openenv/core/containers/test_local_docker_provider.py
|
| 127 |
+
src/openenv/core/containers/runtime/__init__.py
|
| 128 |
+
src/openenv/core/containers/runtime/providers.py
|
| 129 |
+
src/openenv/core/containers/runtime/uv_provider.py
|
| 130 |
+
src/openenv/core/env_server/__init__.py
|
| 131 |
+
src/openenv/core/env_server/base_transforms.py
|
| 132 |
+
src/openenv/core/env_server/exceptions.py
|
| 133 |
+
src/openenv/core/env_server/http_server.py
|
| 134 |
+
src/openenv/core/env_server/interfaces.py
|
| 135 |
+
src/openenv/core/env_server/route_config.py
|
| 136 |
+
src/openenv/core/env_server/serialization.py
|
| 137 |
+
src/openenv/core/env_server/types.py
|
| 138 |
+
src/openenv/core/env_server/web_interface.py
|
| 139 |
+
src/openenv/core/tools/__init__.py
|
| 140 |
+
src/openenv/core/tools/git_server_client.py
|
| 141 |
+
src/openenv/core/tools/local_python_executor.py
|
| 142 |
+
src/openenv_core/__init__.py
|
src/openenv.egg-info/dependency_links.txt
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
|
src/openenv.egg-info/entry_points.txt
ADDED
|
@@ -0,0 +1,2 @@
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[console_scripts]
|
| 2 |
+
openenv = openenv.cli.__main__:main
|
src/openenv.egg-info/requires.txt
ADDED
|
@@ -0,0 +1,32 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
fastapi>=0.104.0
|
| 2 |
+
pydantic>=2.0.0
|
| 3 |
+
uvicorn>=0.24.0
|
| 4 |
+
requests>=2.25.0
|
| 5 |
+
typer>=0.9.0
|
| 6 |
+
rich>=13.0.0
|
| 7 |
+
pyyaml>=6.0
|
| 8 |
+
huggingface_hub>=0.20.0
|
| 9 |
+
openai>=2.7.2
|
| 10 |
+
tomli>=2.3.0
|
| 11 |
+
tomli-w>=1.2.0
|
| 12 |
+
websockets>=15.0.1
|
| 13 |
+
|
| 14 |
+
[all]
|
| 15 |
+
openenv[core]
|
| 16 |
+
openenv[cli]
|
| 17 |
+
|
| 18 |
+
[cli]
|
| 19 |
+
typer>=0.9.0
|
| 20 |
+
rich>=13.0.0
|
| 21 |
+
pyyaml>=6.0
|
| 22 |
+
huggingface_hub>=0.20.0
|
| 23 |
+
openai>=2.7.2
|
| 24 |
+
tomli>=2.3.0
|
| 25 |
+
tomli-w>=1.2.0
|
| 26 |
+
|
| 27 |
+
[core]
|
| 28 |
+
fastapi>=0.104.0
|
| 29 |
+
pydantic>=2.0.0
|
| 30 |
+
uvicorn>=0.24.0
|
| 31 |
+
requests>=2.25.0
|
| 32 |
+
websockets>=15.0.1
|
src/openenv.egg-info/top_level.txt
ADDED
|
@@ -0,0 +1,2 @@
|
|
|
|
|
|
|
|
|
|
| 1 |
+
openenv
|
| 2 |
+
openenv_core
|
src/openenv/__init__.py
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Unified OpenEnv package bundling the CLI and core runtime.
|
| 3 |
+
"""
|
| 4 |
+
|
| 5 |
+
from importlib import metadata
|
| 6 |
+
|
| 7 |
+
from .auto import AutoAction, AutoEnv
|
| 8 |
+
from .core import GenericEnvClient, GenericAction, SyncEnvClient
|
| 9 |
+
|
| 10 |
+
__all__ = [
|
| 11 |
+
"core",
|
| 12 |
+
"cli",
|
| 13 |
+
"AutoEnv",
|
| 14 |
+
"AutoAction",
|
| 15 |
+
"GenericEnvClient",
|
| 16 |
+
"GenericAction",
|
| 17 |
+
"SyncEnvClient",
|
| 18 |
+
]
|
| 19 |
+
|
| 20 |
+
try:
|
| 21 |
+
__version__ = metadata.version("openenv") # type: ignore[arg-type]
|
| 22 |
+
except metadata.PackageNotFoundError: # pragma: no cover - local dev
|
| 23 |
+
__version__ = "0.0.0"
|
src/openenv/auto/__init__.py
ADDED
|
@@ -0,0 +1,39 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Meta Platforms, Inc. and affiliates.
|
| 2 |
+
# All rights reserved.
|
| 3 |
+
#
|
| 4 |
+
# This source code is licensed under the BSD-style license found in the
|
| 5 |
+
# LICENSE file in the root directory of this source tree.
|
| 6 |
+
|
| 7 |
+
"""
|
| 8 |
+
OpenEnv Auto Module
|
| 9 |
+
===================
|
| 10 |
+
|
| 11 |
+
Provides HuggingFace-style auto-discovery API for OpenEnv environments.
|
| 12 |
+
|
| 13 |
+
This module enables automatic environment and action class loading without
|
| 14 |
+
manual imports:
|
| 15 |
+
|
| 16 |
+
>>> from openenv import AutoEnv, AutoAction
|
| 17 |
+
>>>
|
| 18 |
+
>>> # Load environment from installed package or HuggingFace Hub
|
| 19 |
+
>>> env = AutoEnv.from_name("coding-env")
|
| 20 |
+
>>>
|
| 21 |
+
>>> # Get action class
|
| 22 |
+
>>> CodeAction = AutoAction.from_name("coding")
|
| 23 |
+
>>> action = CodeAction(code="print('Hello!')")
|
| 24 |
+
|
| 25 |
+
Classes:
|
| 26 |
+
AutoEnv: Automatic environment client selection and instantiation
|
| 27 |
+
AutoAction: Automatic action class selection
|
| 28 |
+
|
| 29 |
+
The auto-discovery system works by:
|
| 30 |
+
1. Discovering installed openenv-* packages via importlib.metadata
|
| 31 |
+
2. Loading environment manifests (openenv.yaml) from package resources
|
| 32 |
+
3. Supporting HuggingFace Hub repositories for remote environments
|
| 33 |
+
4. Caching discovery results for performance
|
| 34 |
+
"""
|
| 35 |
+
|
| 36 |
+
from .auto_action import AutoAction
|
| 37 |
+
from .auto_env import AutoEnv
|
| 38 |
+
|
| 39 |
+
__all__ = ["AutoEnv", "AutoAction"]
|
src/openenv/auto/_discovery.py
ADDED
|
@@ -0,0 +1,584 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Meta Platforms, Inc. and affiliates.
|
| 2 |
+
# All rights reserved.
|
| 3 |
+
#
|
| 4 |
+
# This source code is licensed under the BSD-style license found in the
|
| 5 |
+
# LICENSE file in the root directory of this source tree.
|
| 6 |
+
|
| 7 |
+
"""
|
| 8 |
+
Environment Auto-Discovery System
|
| 9 |
+
==================================
|
| 10 |
+
|
| 11 |
+
This module provides automatic discovery of OpenEnv environments by:
|
| 12 |
+
1. Discovering installed openenv-* packages using importlib.metadata
|
| 13 |
+
2. Loading manifests (openenv.yaml) from package resources
|
| 14 |
+
3. Caching results for performance
|
| 15 |
+
4. Supporting HuggingFace Hub downloads
|
| 16 |
+
|
| 17 |
+
This enables AutoEnv to work without coupling to src/envs/ directory.
|
| 18 |
+
"""
|
| 19 |
+
|
| 20 |
+
import importlib
|
| 21 |
+
import importlib.metadata
|
| 22 |
+
import importlib.resources
|
| 23 |
+
import json
|
| 24 |
+
import logging
|
| 25 |
+
import re
|
| 26 |
+
import tempfile
|
| 27 |
+
from dataclasses import dataclass, asdict
|
| 28 |
+
from pathlib import Path
|
| 29 |
+
from typing import Dict, Optional, Type, Any
|
| 30 |
+
|
| 31 |
+
import yaml
|
| 32 |
+
|
| 33 |
+
logger = logging.getLogger(__name__)
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
@dataclass
|
| 37 |
+
class EnvironmentInfo:
|
| 38 |
+
"""
|
| 39 |
+
Rich information about a discovered environment.
|
| 40 |
+
|
| 41 |
+
Attributes:
|
| 42 |
+
env_key: Environment key (e.g., "echo", "coding")
|
| 43 |
+
name: Full environment name (e.g., "echo_env")
|
| 44 |
+
package_name: Package name (e.g., "openenv-echo_env")
|
| 45 |
+
version: Version string
|
| 46 |
+
description: Human-readable description
|
| 47 |
+
client_module_path: Full module path to client (e.g., "echo_env.client")
|
| 48 |
+
client_class_name: Client class name (e.g., "EchoEnv")
|
| 49 |
+
action_class_name: Action class name (e.g., "EchoAction")
|
| 50 |
+
observation_class_name: Observation class name (e.g., "EchoObservation")
|
| 51 |
+
default_image: Default Docker image name (e.g., "echo-env:latest")
|
| 52 |
+
spec_version: OpenEnv spec version (from openenv.yaml)
|
| 53 |
+
manifest: Original manifest data
|
| 54 |
+
"""
|
| 55 |
+
|
| 56 |
+
env_key: str
|
| 57 |
+
name: str
|
| 58 |
+
package_name: str
|
| 59 |
+
version: str
|
| 60 |
+
description: str
|
| 61 |
+
client_module_path: str
|
| 62 |
+
client_class_name: str
|
| 63 |
+
action_class_name: str
|
| 64 |
+
observation_class_name: str
|
| 65 |
+
default_image: str
|
| 66 |
+
spec_version: Optional[int] = None
|
| 67 |
+
manifest: Optional[Dict[str, Any]] = None
|
| 68 |
+
|
| 69 |
+
def get_client_class(self) -> Type:
|
| 70 |
+
"""
|
| 71 |
+
Dynamically import and return the client class.
|
| 72 |
+
|
| 73 |
+
Returns:
|
| 74 |
+
Client class (e.g., EchoEnv)
|
| 75 |
+
|
| 76 |
+
Raises:
|
| 77 |
+
ImportError: If module or class cannot be imported
|
| 78 |
+
"""
|
| 79 |
+
try:
|
| 80 |
+
module = importlib.import_module(self.client_module_path)
|
| 81 |
+
return getattr(module, self.client_class_name)
|
| 82 |
+
except ImportError as e:
|
| 83 |
+
raise ImportError(
|
| 84 |
+
f"Failed to import {self.client_class_name} from {self.client_module_path}: {e}\n"
|
| 85 |
+
f"Make sure the package '{self.package_name}' is installed: "
|
| 86 |
+
f"pip install {self.package_name}"
|
| 87 |
+
) from e
|
| 88 |
+
except AttributeError as e:
|
| 89 |
+
raise ImportError(
|
| 90 |
+
f"Class {self.client_class_name} not found in {self.client_module_path}: {e}"
|
| 91 |
+
) from e
|
| 92 |
+
|
| 93 |
+
def get_action_class(self) -> Type:
|
| 94 |
+
"""
|
| 95 |
+
Dynamically import and return the action class.
|
| 96 |
+
|
| 97 |
+
Returns:
|
| 98 |
+
Action class (e.g., EchoAction)
|
| 99 |
+
|
| 100 |
+
Raises:
|
| 101 |
+
ImportError: If module or class cannot be imported
|
| 102 |
+
"""
|
| 103 |
+
try:
|
| 104 |
+
module = importlib.import_module(self.client_module_path)
|
| 105 |
+
return getattr(module, self.action_class_name)
|
| 106 |
+
except ImportError as e:
|
| 107 |
+
raise ImportError(
|
| 108 |
+
f"Failed to import {self.action_class_name} from {self.client_module_path}: {e}\n"
|
| 109 |
+
f"Make sure the package '{self.package_name}' is installed: "
|
| 110 |
+
f"pip install {self.package_name}"
|
| 111 |
+
) from e
|
| 112 |
+
except AttributeError as e:
|
| 113 |
+
raise ImportError(
|
| 114 |
+
f"Class {self.action_class_name} not found in {self.client_module_path}: {e}"
|
| 115 |
+
) from e
|
| 116 |
+
|
| 117 |
+
def get_observation_class(self) -> Type:
|
| 118 |
+
"""
|
| 119 |
+
Dynamically import and return the observation class.
|
| 120 |
+
|
| 121 |
+
Returns:
|
| 122 |
+
Observation class (e.g., EchoObservation)
|
| 123 |
+
|
| 124 |
+
Raises:
|
| 125 |
+
ImportError: If module or class cannot be imported
|
| 126 |
+
"""
|
| 127 |
+
try:
|
| 128 |
+
module = importlib.import_module(self.client_module_path)
|
| 129 |
+
return getattr(module, self.observation_class_name)
|
| 130 |
+
except ImportError as e:
|
| 131 |
+
raise ImportError(
|
| 132 |
+
f"Failed to import {self.observation_class_name} from {self.client_module_path}: {e}\n"
|
| 133 |
+
f"Make sure the package '{self.package_name}' is installed: "
|
| 134 |
+
f"pip install {self.package_name}"
|
| 135 |
+
) from e
|
| 136 |
+
except AttributeError as e:
|
| 137 |
+
raise ImportError(
|
| 138 |
+
f"Class {self.observation_class_name} not found in {self.client_module_path}: {e}"
|
| 139 |
+
) from e
|
| 140 |
+
|
| 141 |
+
|
| 142 |
+
def _normalize_env_name(name: str) -> str:
|
| 143 |
+
"""
|
| 144 |
+
Normalize environment name to standard format.
|
| 145 |
+
|
| 146 |
+
Args:
|
| 147 |
+
name: Input name (e.g., "echo", "echo-env", "echo_env")
|
| 148 |
+
|
| 149 |
+
Returns:
|
| 150 |
+
Normalized name (e.g., "echo_env")
|
| 151 |
+
|
| 152 |
+
Examples:
|
| 153 |
+
>>> _normalize_env_name("echo")
|
| 154 |
+
'echo_env'
|
| 155 |
+
>>> _normalize_env_name("echo-env")
|
| 156 |
+
'echo_env'
|
| 157 |
+
>>> _normalize_env_name("echo_env")
|
| 158 |
+
'echo_env'
|
| 159 |
+
"""
|
| 160 |
+
# Remove common suffixes
|
| 161 |
+
name = re.sub(r"[-_]env$", "", name)
|
| 162 |
+
# Convert hyphens to underscores
|
| 163 |
+
name = name.replace("-", "_")
|
| 164 |
+
# Add _env suffix if not present
|
| 165 |
+
if not name.endswith("_env"):
|
| 166 |
+
name = f"{name}_env"
|
| 167 |
+
return name
|
| 168 |
+
|
| 169 |
+
|
| 170 |
+
def _is_hub_url(name: str) -> bool:
|
| 171 |
+
"""
|
| 172 |
+
Check if name is a HuggingFace Hub URL or repo ID.
|
| 173 |
+
|
| 174 |
+
Args:
|
| 175 |
+
name: Input name
|
| 176 |
+
|
| 177 |
+
Returns:
|
| 178 |
+
True if it looks like a Hub URL
|
| 179 |
+
|
| 180 |
+
Examples:
|
| 181 |
+
>>> _is_hub_url("meta-pytorch/echo_env")
|
| 182 |
+
True
|
| 183 |
+
>>> _is_hub_url("https://huggingface.co/meta-pytorch/echo_env")
|
| 184 |
+
True
|
| 185 |
+
>>> _is_hub_url("echo")
|
| 186 |
+
False
|
| 187 |
+
"""
|
| 188 |
+
# Contains org/repo pattern or huggingface.co domain
|
| 189 |
+
return "/" in name or "huggingface.co" in name
|
| 190 |
+
|
| 191 |
+
|
| 192 |
+
def _infer_class_name(env_name: str, class_type: str) -> str:
|
| 193 |
+
"""
|
| 194 |
+
Infer class name from environment name using simple conventions.
|
| 195 |
+
|
| 196 |
+
Args:
|
| 197 |
+
env_name: Environment name (e.g., "echo_env")
|
| 198 |
+
class_type: Type of class ("client", "action", "observation")
|
| 199 |
+
|
| 200 |
+
Returns:
|
| 201 |
+
Inferred class name
|
| 202 |
+
|
| 203 |
+
Examples:
|
| 204 |
+
>>> _infer_class_name("echo_env", "client")
|
| 205 |
+
'EchoEnv'
|
| 206 |
+
>>> _infer_class_name("echo_env", "action")
|
| 207 |
+
'EchoAction'
|
| 208 |
+
"""
|
| 209 |
+
# Remove _env suffix for base name
|
| 210 |
+
base_name = env_name.replace("_env", "")
|
| 211 |
+
|
| 212 |
+
# Convert to PascalCase
|
| 213 |
+
pascal_name = "".join(word.capitalize() for word in base_name.split("_"))
|
| 214 |
+
|
| 215 |
+
# Add suffix based on type
|
| 216 |
+
if class_type == "client":
|
| 217 |
+
return f"{pascal_name}Env"
|
| 218 |
+
elif class_type == "action":
|
| 219 |
+
return f"{pascal_name}Action"
|
| 220 |
+
elif class_type == "observation":
|
| 221 |
+
return f"{pascal_name}Observation"
|
| 222 |
+
else:
|
| 223 |
+
raise ValueError(f"Unknown class type: {class_type}")
|
| 224 |
+
|
| 225 |
+
|
| 226 |
+
def _load_manifest_from_package(
|
| 227 |
+
package_name: str, module_name: str
|
| 228 |
+
) -> Optional[Dict[str, Any]]:
|
| 229 |
+
"""
|
| 230 |
+
Load openenv.yaml manifest from an installed package.
|
| 231 |
+
|
| 232 |
+
Args:
|
| 233 |
+
package_name: Package name (e.g., "openenv-echo_env")
|
| 234 |
+
module_name: Module name (e.g., "echo_env")
|
| 235 |
+
|
| 236 |
+
Returns:
|
| 237 |
+
Parsed manifest dictionary, or None if not found
|
| 238 |
+
|
| 239 |
+
"""
|
| 240 |
+
try:
|
| 241 |
+
# Try to read openenv.yaml from package
|
| 242 |
+
if hasattr(importlib.resources, "files"):
|
| 243 |
+
# Python 3.9+
|
| 244 |
+
package_files = importlib.resources.files(module_name)
|
| 245 |
+
if (package_files / "openenv.yaml").is_file():
|
| 246 |
+
manifest_text = (package_files / "openenv.yaml").read_text()
|
| 247 |
+
return yaml.safe_load(manifest_text)
|
| 248 |
+
else:
|
| 249 |
+
# Python 3.7-3.8 fallback
|
| 250 |
+
with importlib.resources.open_text(module_name, "openenv.yaml") as f:
|
| 251 |
+
return yaml.safe_load(f)
|
| 252 |
+
except (FileNotFoundError, ModuleNotFoundError, AttributeError):
|
| 253 |
+
logger.debug(f"No openenv.yaml found in {module_name}")
|
| 254 |
+
return None
|
| 255 |
+
except Exception as e:
|
| 256 |
+
logger.warning(f"Failed to load openenv.yaml from {module_name}: {e}")
|
| 257 |
+
return None
|
| 258 |
+
|
| 259 |
+
|
| 260 |
+
def _create_env_info_from_package(
|
| 261 |
+
package_name: str, module_name: str, version: str
|
| 262 |
+
) -> Optional[EnvironmentInfo]:
|
| 263 |
+
"""
|
| 264 |
+
Create EnvironmentInfo from an installed package.
|
| 265 |
+
|
| 266 |
+
Args:
|
| 267 |
+
package_name: Package name (e.g., "openenv-echo_env")
|
| 268 |
+
module_name: Module name (e.g., "echo_env")
|
| 269 |
+
version: Package version
|
| 270 |
+
|
| 271 |
+
Returns:
|
| 272 |
+
EnvironmentInfo instance, or None if invalid
|
| 273 |
+
"""
|
| 274 |
+
# Load manifest
|
| 275 |
+
manifest = _load_manifest_from_package(package_name, module_name)
|
| 276 |
+
|
| 277 |
+
# Get environment name
|
| 278 |
+
if manifest and "name" in manifest:
|
| 279 |
+
env_name = manifest["name"]
|
| 280 |
+
else:
|
| 281 |
+
# Infer from module name
|
| 282 |
+
env_name = module_name
|
| 283 |
+
|
| 284 |
+
# Normalize to ensure _env suffix
|
| 285 |
+
if not env_name.endswith("_env"):
|
| 286 |
+
env_name = f"{env_name}_env"
|
| 287 |
+
|
| 288 |
+
# Determine env_key (e.g., "echo_env" → "echo")
|
| 289 |
+
env_key = env_name.replace("_env", "") if env_name.endswith("_env") else env_name
|
| 290 |
+
|
| 291 |
+
# Get description
|
| 292 |
+
description = (
|
| 293 |
+
manifest.get("description", f"{env_name} environment")
|
| 294 |
+
if manifest
|
| 295 |
+
else f"{env_name} environment"
|
| 296 |
+
)
|
| 297 |
+
|
| 298 |
+
# Get spec version
|
| 299 |
+
spec_version = manifest.get("spec_version") if manifest else None
|
| 300 |
+
|
| 301 |
+
# Determine class names
|
| 302 |
+
# Check if manifest has custom class names (custom format)
|
| 303 |
+
if manifest and "action" in manifest and "observation" in manifest:
|
| 304 |
+
# Custom format (like coding_env)
|
| 305 |
+
client_class_name = _infer_class_name(env_name, "client")
|
| 306 |
+
action_class_name = manifest.get(
|
| 307 |
+
"action", _infer_class_name(env_name, "action")
|
| 308 |
+
)
|
| 309 |
+
observation_class_name = manifest.get(
|
| 310 |
+
"observation", _infer_class_name(env_name, "observation")
|
| 311 |
+
)
|
| 312 |
+
else:
|
| 313 |
+
# Use conventions
|
| 314 |
+
client_class_name = _infer_class_name(env_name, "client")
|
| 315 |
+
action_class_name = _infer_class_name(env_name, "action")
|
| 316 |
+
observation_class_name = _infer_class_name(env_name, "observation")
|
| 317 |
+
|
| 318 |
+
# Module path is just module_name.client
|
| 319 |
+
client_module_path = f"{module_name}.client"
|
| 320 |
+
|
| 321 |
+
# Determine default Docker image name
|
| 322 |
+
image_name = env_name.replace("_", "-")
|
| 323 |
+
default_image = f"{image_name}:latest"
|
| 324 |
+
|
| 325 |
+
return EnvironmentInfo(
|
| 326 |
+
env_key=env_key,
|
| 327 |
+
name=env_name,
|
| 328 |
+
package_name=package_name,
|
| 329 |
+
version=version,
|
| 330 |
+
description=description,
|
| 331 |
+
client_module_path=client_module_path,
|
| 332 |
+
client_class_name=client_class_name,
|
| 333 |
+
action_class_name=action_class_name,
|
| 334 |
+
observation_class_name=observation_class_name,
|
| 335 |
+
default_image=default_image,
|
| 336 |
+
spec_version=spec_version,
|
| 337 |
+
manifest=manifest,
|
| 338 |
+
)
|
| 339 |
+
|
| 340 |
+
|
| 341 |
+
class EnvironmentDiscovery:
|
| 342 |
+
"""
|
| 343 |
+
Auto-discovery system for OpenEnv environments using installed packages.
|
| 344 |
+
|
| 345 |
+
This class discovers installed openenv-* packages and loads their metadata.
|
| 346 |
+
"""
|
| 347 |
+
|
| 348 |
+
def __init__(self):
|
| 349 |
+
"""Initialize discovery system."""
|
| 350 |
+
self._cache: Optional[Dict[str, EnvironmentInfo]] = None
|
| 351 |
+
self._cache_file = Path(tempfile.gettempdir()) / "openenv_discovery_cache.json"
|
| 352 |
+
|
| 353 |
+
def _discover_installed_packages(self) -> Dict[str, EnvironmentInfo]:
|
| 354 |
+
"""
|
| 355 |
+
Discover all installed openenv-* packages.
|
| 356 |
+
|
| 357 |
+
Returns:
|
| 358 |
+
Dictionary mapping env_key to EnvironmentInfo
|
| 359 |
+
"""
|
| 360 |
+
environments = {}
|
| 361 |
+
|
| 362 |
+
# Invalidate import caches to ensure we pick up newly installed packages
|
| 363 |
+
importlib.invalidate_caches()
|
| 364 |
+
|
| 365 |
+
# Get all installed packages
|
| 366 |
+
try:
|
| 367 |
+
distributions = importlib.metadata.distributions()
|
| 368 |
+
except Exception as e:
|
| 369 |
+
logger.warning(f"Failed to get installed packages: {e}")
|
| 370 |
+
return environments
|
| 371 |
+
|
| 372 |
+
# Filter for openenv-* packages (exclude openenv-core)
|
| 373 |
+
for dist in distributions:
|
| 374 |
+
package_name = dist.metadata["Name"]
|
| 375 |
+
|
| 376 |
+
if not package_name.startswith("openenv-"):
|
| 377 |
+
continue
|
| 378 |
+
|
| 379 |
+
if package_name == "openenv-core":
|
| 380 |
+
continue
|
| 381 |
+
|
| 382 |
+
# Get module name (e.g., "openenv-echo_env" → "echo_env")
|
| 383 |
+
module_name = package_name.replace("openenv-", "").replace("-", "_")
|
| 384 |
+
|
| 385 |
+
# Get version
|
| 386 |
+
version = dist.version
|
| 387 |
+
|
| 388 |
+
try:
|
| 389 |
+
# Create environment info
|
| 390 |
+
env_info = _create_env_info_from_package(
|
| 391 |
+
package_name, module_name, version
|
| 392 |
+
)
|
| 393 |
+
|
| 394 |
+
if env_info:
|
| 395 |
+
environments[env_info.env_key] = env_info
|
| 396 |
+
logger.debug(
|
| 397 |
+
f"Discovered environment: {env_info.env_key} ({package_name})"
|
| 398 |
+
)
|
| 399 |
+
|
| 400 |
+
except Exception as e:
|
| 401 |
+
logger.warning(f"Failed to load environment from {package_name}: {e}")
|
| 402 |
+
continue
|
| 403 |
+
|
| 404 |
+
return environments
|
| 405 |
+
|
| 406 |
+
def _load_cache(self) -> Optional[Dict[str, EnvironmentInfo]]:
|
| 407 |
+
"""
|
| 408 |
+
Load cached discovery results.
|
| 409 |
+
|
| 410 |
+
Returns:
|
| 411 |
+
Dictionary of env_key -> EnvironmentInfo, or None if cache invalid
|
| 412 |
+
"""
|
| 413 |
+
if not self._cache_file.exists():
|
| 414 |
+
return None
|
| 415 |
+
|
| 416 |
+
try:
|
| 417 |
+
with open(self._cache_file, "r") as f:
|
| 418 |
+
cache_data = json.load(f)
|
| 419 |
+
|
| 420 |
+
# Reconstruct EnvironmentInfo objects
|
| 421 |
+
cache = {}
|
| 422 |
+
for env_key, env_data in cache_data.items():
|
| 423 |
+
cache[env_key] = EnvironmentInfo(**env_data)
|
| 424 |
+
|
| 425 |
+
return cache
|
| 426 |
+
except Exception as e:
|
| 427 |
+
logger.warning(f"Failed to load discovery cache: {e}")
|
| 428 |
+
return None
|
| 429 |
+
|
| 430 |
+
def _save_cache(self, environments: Dict[str, EnvironmentInfo]) -> None:
|
| 431 |
+
"""
|
| 432 |
+
Save discovery results to cache.
|
| 433 |
+
|
| 434 |
+
Args:
|
| 435 |
+
environments: Dictionary of env_key -> EnvironmentInfo
|
| 436 |
+
"""
|
| 437 |
+
try:
|
| 438 |
+
cache_data = {}
|
| 439 |
+
for env_key, env_info in environments.items():
|
| 440 |
+
cache_data[env_key] = asdict(env_info)
|
| 441 |
+
|
| 442 |
+
with open(self._cache_file, "w") as f:
|
| 443 |
+
json.dump(cache_data, f, indent=2)
|
| 444 |
+
|
| 445 |
+
except Exception as e:
|
| 446 |
+
logger.warning(f"Failed to save discovery cache: {e}")
|
| 447 |
+
|
| 448 |
+
def discover(self, use_cache: bool = True) -> Dict[str, EnvironmentInfo]:
|
| 449 |
+
"""
|
| 450 |
+
Discover all installed OpenEnv environments.
|
| 451 |
+
|
| 452 |
+
Args:
|
| 453 |
+
use_cache: If True, try to load from cache first
|
| 454 |
+
|
| 455 |
+
Returns:
|
| 456 |
+
Dictionary mapping env_key to EnvironmentInfo
|
| 457 |
+
|
| 458 |
+
Examples:
|
| 459 |
+
>>> discovery = EnvironmentDiscovery()
|
| 460 |
+
>>> envs = discovery.discover()
|
| 461 |
+
>>> print(envs.keys())
|
| 462 |
+
dict_keys(['echo', 'coding', ...])
|
| 463 |
+
"""
|
| 464 |
+
# Try to load from memory cache first
|
| 465 |
+
if use_cache and self._cache is not None:
|
| 466 |
+
return self._cache
|
| 467 |
+
|
| 468 |
+
# Try to load from file cache
|
| 469 |
+
if use_cache:
|
| 470 |
+
cached = self._load_cache()
|
| 471 |
+
if cached is not None:
|
| 472 |
+
self._cache = cached
|
| 473 |
+
return self._cache
|
| 474 |
+
|
| 475 |
+
# Discover from installed packages
|
| 476 |
+
environments = self._discover_installed_packages()
|
| 477 |
+
|
| 478 |
+
# Save to cache
|
| 479 |
+
self._save_cache(environments)
|
| 480 |
+
self._cache = environments
|
| 481 |
+
|
| 482 |
+
return environments
|
| 483 |
+
|
| 484 |
+
def get_environment(self, env_key: str) -> Optional[EnvironmentInfo]:
|
| 485 |
+
"""
|
| 486 |
+
Get information about a specific environment.
|
| 487 |
+
|
| 488 |
+
Args:
|
| 489 |
+
env_key: Environment key (e.g., "echo", "coding")
|
| 490 |
+
|
| 491 |
+
Returns:
|
| 492 |
+
EnvironmentInfo if found, None otherwise
|
| 493 |
+
|
| 494 |
+
Examples:
|
| 495 |
+
>>> discovery = EnvironmentDiscovery()
|
| 496 |
+
>>> env = discovery.get_environment("echo")
|
| 497 |
+
>>> print(env.client_class_name)
|
| 498 |
+
'EchoEnv'
|
| 499 |
+
"""
|
| 500 |
+
environments = self.discover()
|
| 501 |
+
return environments.get(env_key)
|
| 502 |
+
|
| 503 |
+
def get_environment_by_name(self, name: str) -> Optional[EnvironmentInfo]:
|
| 504 |
+
"""
|
| 505 |
+
Get environment info by flexible name matching.
|
| 506 |
+
|
| 507 |
+
Args:
|
| 508 |
+
name: Environment name (e.g., "echo", "echo-env", "echo_env")
|
| 509 |
+
|
| 510 |
+
Returns:
|
| 511 |
+
EnvironmentInfo if found, None otherwise
|
| 512 |
+
"""
|
| 513 |
+
# Normalize name to env_key
|
| 514 |
+
normalized = _normalize_env_name(name)
|
| 515 |
+
env_key = normalized.replace("_env", "")
|
| 516 |
+
|
| 517 |
+
return self.get_environment(env_key)
|
| 518 |
+
|
| 519 |
+
def list_environments(self) -> None:
|
| 520 |
+
"""
|
| 521 |
+
Print a formatted list of all discovered environments.
|
| 522 |
+
|
| 523 |
+
Examples:
|
| 524 |
+
>>> discovery = EnvironmentDiscovery()
|
| 525 |
+
>>> discovery.list_environments()
|
| 526 |
+
Available OpenEnv Environments:
|
| 527 |
+
----------------------------------------------------------------------
|
| 528 |
+
echo : Echo Environment (v0.1.0) - openenv-echo_env
|
| 529 |
+
coding : Coding Environment (v0.1.0) - openenv-coding_env
|
| 530 |
+
...
|
| 531 |
+
"""
|
| 532 |
+
environments = self.discover()
|
| 533 |
+
|
| 534 |
+
print("Available OpenEnv Environments:")
|
| 535 |
+
print("-" * 70)
|
| 536 |
+
|
| 537 |
+
if not environments:
|
| 538 |
+
print(" No OpenEnv environments found.")
|
| 539 |
+
print(" Install environments with: pip install openenv-<env-name>")
|
| 540 |
+
else:
|
| 541 |
+
for env_key in sorted(environments.keys()):
|
| 542 |
+
env = environments[env_key]
|
| 543 |
+
print(f" {env_key:<15}: {env.description} (v{env.version})")
|
| 544 |
+
print(f" Package: {env.package_name}")
|
| 545 |
+
|
| 546 |
+
print("-" * 70)
|
| 547 |
+
print(f"Total: {len(environments)} environments")
|
| 548 |
+
|
| 549 |
+
def clear_cache(self) -> None:
|
| 550 |
+
"""Clear the discovery cache."""
|
| 551 |
+
if self._cache_file.exists():
|
| 552 |
+
self._cache_file.unlink()
|
| 553 |
+
self._cache = None
|
| 554 |
+
|
| 555 |
+
|
| 556 |
+
# Global discovery instance
|
| 557 |
+
_global_discovery: Optional[EnvironmentDiscovery] = None
|
| 558 |
+
|
| 559 |
+
|
| 560 |
+
def get_discovery() -> EnvironmentDiscovery:
|
| 561 |
+
"""
|
| 562 |
+
Get or create the global discovery instance.
|
| 563 |
+
|
| 564 |
+
Returns:
|
| 565 |
+
Global EnvironmentDiscovery instance
|
| 566 |
+
|
| 567 |
+
Examples:
|
| 568 |
+
>>> discovery = get_discovery()
|
| 569 |
+
>>> envs = discovery.discover()
|
| 570 |
+
"""
|
| 571 |
+
global _global_discovery
|
| 572 |
+
|
| 573 |
+
if _global_discovery is None:
|
| 574 |
+
_global_discovery = EnvironmentDiscovery()
|
| 575 |
+
|
| 576 |
+
return _global_discovery
|
| 577 |
+
|
| 578 |
+
|
| 579 |
+
def reset_discovery() -> None:
|
| 580 |
+
"""Reset the global discovery instance (useful for testing)."""
|
| 581 |
+
global _global_discovery
|
| 582 |
+
if _global_discovery is not None:
|
| 583 |
+
_global_discovery.clear_cache()
|
| 584 |
+
_global_discovery = None
|
src/openenv/auto/auto_action.py
ADDED
|
@@ -0,0 +1,276 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Meta Platforms, Inc. and affiliates.
|
| 2 |
+
# All rights reserved.
|
| 3 |
+
#
|
| 4 |
+
# This source code is licensed under the BSD-style license found in the
|
| 5 |
+
# LICENSE file in the root directory of this source tree.
|
| 6 |
+
|
| 7 |
+
"""
|
| 8 |
+
AutoAction - Automatic Action Class Selection
|
| 9 |
+
==============================================
|
| 10 |
+
|
| 11 |
+
AutoAction provides a HuggingFace-style API for automatically retrieving the
|
| 12 |
+
correct Action class from installed packages or HuggingFace Hub.
|
| 13 |
+
|
| 14 |
+
This module simplifies working with environment actions by automatically
|
| 15 |
+
detecting and returning the appropriate Action class without requiring
|
| 16 |
+
manual imports.
|
| 17 |
+
|
| 18 |
+
Example:
|
| 19 |
+
>>> from openenv import AutoEnv, AutoAction
|
| 20 |
+
>>>
|
| 21 |
+
>>> # Get Action class from environment name
|
| 22 |
+
>>> CodeAction = AutoAction.from_env("coding")
|
| 23 |
+
>>> action = CodeAction(code="print('Hello!')")
|
| 24 |
+
>>>
|
| 25 |
+
>>> # From HuggingFace Hub
|
| 26 |
+
>>> CodeAction = AutoAction.from_env("meta-pytorch/coding-env")
|
| 27 |
+
>>>
|
| 28 |
+
>>> # Use with AutoEnv
|
| 29 |
+
>>> env = AutoEnv.from_env("coding-env")
|
| 30 |
+
>>> result = env.step(action)
|
| 31 |
+
"""
|
| 32 |
+
|
| 33 |
+
from __future__ import annotations
|
| 34 |
+
|
| 35 |
+
import logging
|
| 36 |
+
from typing import Type, Dict, Any
|
| 37 |
+
|
| 38 |
+
from ._discovery import get_discovery, _is_hub_url
|
| 39 |
+
from .auto_env import AutoEnv
|
| 40 |
+
|
| 41 |
+
logger = logging.getLogger(__name__)
|
| 42 |
+
|
| 43 |
+
|
| 44 |
+
class AutoAction:
|
| 45 |
+
"""
|
| 46 |
+
AutoAction automatically retrieves the correct Action class based on
|
| 47 |
+
environment names or HuggingFace Hub repositories.
|
| 48 |
+
|
| 49 |
+
This class follows the HuggingFace AutoModel pattern, making it easy to
|
| 50 |
+
get the right Action class without needing to know which module to import.
|
| 51 |
+
|
| 52 |
+
The class provides factory methods that look up the Action class and
|
| 53 |
+
return the class (not an instance) for you to instantiate.
|
| 54 |
+
|
| 55 |
+
Example:
|
| 56 |
+
>>> # From installed package
|
| 57 |
+
>>> CodeAction = AutoAction.from_env("coding")
|
| 58 |
+
>>> action = CodeAction(code="print('test')")
|
| 59 |
+
>>>
|
| 60 |
+
>>> # From HuggingFace Hub
|
| 61 |
+
>>> CodeAction = AutoAction.from_env("meta-pytorch/coding-env")
|
| 62 |
+
>>> action = CodeAction(code="print('test')")
|
| 63 |
+
>>>
|
| 64 |
+
>>> # Use with AutoEnv for a complete workflow
|
| 65 |
+
>>> env = AutoEnv.from_env("coding-env")
|
| 66 |
+
>>> ActionClass = AutoAction.from_env("coding-env")
|
| 67 |
+
>>> action = ActionClass(code="print('Hello, AutoAction!')")
|
| 68 |
+
>>> result = env.step(action)
|
| 69 |
+
|
| 70 |
+
Note:
|
| 71 |
+
AutoAction is not meant to be instantiated directly. Use the class
|
| 72 |
+
method from_env() instead.
|
| 73 |
+
"""
|
| 74 |
+
|
| 75 |
+
def __init__(self):
|
| 76 |
+
"""AutoAction should not be instantiated directly. Use class methods instead."""
|
| 77 |
+
raise TypeError(
|
| 78 |
+
"AutoAction is a factory class and should not be instantiated directly. "
|
| 79 |
+
"Use AutoAction.from_hub() or AutoAction.from_env() instead."
|
| 80 |
+
)
|
| 81 |
+
|
| 82 |
+
@classmethod
|
| 83 |
+
def from_env(cls, name: str, skip_install: bool = False) -> Type:
|
| 84 |
+
"""
|
| 85 |
+
Get the Action class from environment name or HuggingFace Hub repository.
|
| 86 |
+
|
| 87 |
+
This method automatically:
|
| 88 |
+
1. Checks if the name is a HuggingFace Hub URL/repo ID
|
| 89 |
+
2. If Hub: downloads and installs the environment package
|
| 90 |
+
3. If local: looks up the installed openenv-* package
|
| 91 |
+
4. Imports and returns the Action class
|
| 92 |
+
|
| 93 |
+
Args:
|
| 94 |
+
name: Environment name or HuggingFace Hub repo ID
|
| 95 |
+
Examples:
|
| 96 |
+
- "coding" / "coding-env" / "coding_env"
|
| 97 |
+
- "meta-pytorch/coding-env" (Hub repo ID)
|
| 98 |
+
- "https://huggingface.co/meta-pytorch/coding-env" (Hub URL)
|
| 99 |
+
skip_install: If True, skip package installation and return
|
| 100 |
+
GenericAction class instead. Use this when working with
|
| 101 |
+
GenericEnvClient to avoid installing remote packages.
|
| 102 |
+
|
| 103 |
+
Returns:
|
| 104 |
+
Action class (not an instance!). Returns GenericAction when
|
| 105 |
+
skip_install=True.
|
| 106 |
+
|
| 107 |
+
Raises:
|
| 108 |
+
ValueError: If environment not found (only when skip_install=False)
|
| 109 |
+
ImportError: If environment package is not installed (only when skip_install=False)
|
| 110 |
+
|
| 111 |
+
Examples:
|
| 112 |
+
>>> # From installed package
|
| 113 |
+
>>> CodeAction = AutoAction.from_env("coding-env")
|
| 114 |
+
>>> action = CodeAction(code="print('Hello!')")
|
| 115 |
+
>>>
|
| 116 |
+
>>> # From HuggingFace Hub
|
| 117 |
+
>>> CodeAction = AutoAction.from_env("meta-pytorch/coding-env")
|
| 118 |
+
>>> action = CodeAction(code="print('Hello!')")
|
| 119 |
+
>>>
|
| 120 |
+
>>> # Skip installation, use GenericAction (for GenericEnvClient)
|
| 121 |
+
>>> ActionClass = AutoAction.from_env("user/repo", skip_install=True)
|
| 122 |
+
>>> action = ActionClass(code="print('Hello!')") # Returns GenericAction
|
| 123 |
+
>>>
|
| 124 |
+
>>> # Different name formats
|
| 125 |
+
>>> EchoAction = AutoAction.from_env("echo")
|
| 126 |
+
>>> EchoAction = AutoAction.from_env("echo-env")
|
| 127 |
+
>>> EchoAction = AutoAction.from_env("echo_env")
|
| 128 |
+
"""
|
| 129 |
+
# If skip_install is True, return GenericAction without any package lookup
|
| 130 |
+
if skip_install:
|
| 131 |
+
from openenv.core.generic_client import GenericAction
|
| 132 |
+
|
| 133 |
+
logger.info(
|
| 134 |
+
f"Returning GenericAction for '{name}' (skip_install=True). "
|
| 135 |
+
f"Use keyword arguments to create actions: GenericAction(code='...')"
|
| 136 |
+
)
|
| 137 |
+
return GenericAction
|
| 138 |
+
|
| 139 |
+
# Check if it's a HuggingFace Hub URL or repo ID
|
| 140 |
+
if _is_hub_url(name):
|
| 141 |
+
# Ensure package is installed (reuse AutoEnv logic, downloads only if needed)
|
| 142 |
+
env_name = AutoEnv._ensure_package_from_hub(name)
|
| 143 |
+
else:
|
| 144 |
+
env_name = name
|
| 145 |
+
|
| 146 |
+
# Get environment info from discovery
|
| 147 |
+
discovery = get_discovery()
|
| 148 |
+
env_info = discovery.get_environment_by_name(env_name)
|
| 149 |
+
|
| 150 |
+
if not env_info:
|
| 151 |
+
# Environment not found - provide helpful error message
|
| 152 |
+
available_envs = discovery.discover()
|
| 153 |
+
|
| 154 |
+
if not available_envs:
|
| 155 |
+
raise ValueError(
|
| 156 |
+
"No OpenEnv environments found.\n"
|
| 157 |
+
"Install an environment with: pip install openenv-<env-name>\n"
|
| 158 |
+
"Or specify a HuggingFace Hub repository: AutoAction.from_env('openenv/echo_env')"
|
| 159 |
+
)
|
| 160 |
+
|
| 161 |
+
# Try to suggest similar environment names
|
| 162 |
+
from difflib import get_close_matches
|
| 163 |
+
|
| 164 |
+
env_keys = list(available_envs.keys())
|
| 165 |
+
suggestions = get_close_matches(env_name, env_keys, n=3, cutoff=0.6)
|
| 166 |
+
|
| 167 |
+
error_msg = f"Unknown environment '{env_name}'.\n"
|
| 168 |
+
if suggestions:
|
| 169 |
+
error_msg += f"Did you mean: {', '.join(suggestions)}?\n"
|
| 170 |
+
error_msg += f"Available environments: {', '.join(sorted(env_keys))}"
|
| 171 |
+
|
| 172 |
+
raise ValueError(error_msg)
|
| 173 |
+
|
| 174 |
+
# Get the action class
|
| 175 |
+
try:
|
| 176 |
+
action_class = env_info.get_action_class()
|
| 177 |
+
return action_class
|
| 178 |
+
except ImportError as e:
|
| 179 |
+
raise ImportError(
|
| 180 |
+
f"Failed to import action class for '{env_name}'.\n"
|
| 181 |
+
f"Package '{env_info.package_name}' appears to be installed but the module cannot be imported.\n"
|
| 182 |
+
f"Try reinstalling: pip install --force-reinstall {env_info.package_name}\n"
|
| 183 |
+
f"Original error: {e}"
|
| 184 |
+
) from e
|
| 185 |
+
|
| 186 |
+
@classmethod
|
| 187 |
+
def from_hub(cls, env_name: str, skip_install: bool = False) -> Type:
|
| 188 |
+
"""
|
| 189 |
+
Get the Action class from environment name.
|
| 190 |
+
|
| 191 |
+
This is an alias for from_env() for backward compatibility and clarity.
|
| 192 |
+
|
| 193 |
+
Args:
|
| 194 |
+
env_name: Environment name (e.g., "coding", "echo")
|
| 195 |
+
skip_install: If True, skip package installation and return
|
| 196 |
+
GenericAction class instead.
|
| 197 |
+
|
| 198 |
+
Returns:
|
| 199 |
+
Action class (not an instance!)
|
| 200 |
+
|
| 201 |
+
Examples:
|
| 202 |
+
>>> CodeAction = AutoAction.from_hub("coding")
|
| 203 |
+
>>> action = CodeAction(code="print('Hello!')")
|
| 204 |
+
"""
|
| 205 |
+
return cls.from_env(env_name, skip_install=skip_install)
|
| 206 |
+
|
| 207 |
+
@classmethod
|
| 208 |
+
def get_action_info(cls, name: str) -> Dict[str, Any]:
|
| 209 |
+
"""
|
| 210 |
+
Get detailed information about an action class.
|
| 211 |
+
|
| 212 |
+
Args:
|
| 213 |
+
name: Environment name
|
| 214 |
+
|
| 215 |
+
Returns:
|
| 216 |
+
Dictionary with action class metadata
|
| 217 |
+
|
| 218 |
+
Raises:
|
| 219 |
+
ValueError: If environment not found
|
| 220 |
+
|
| 221 |
+
Examples:
|
| 222 |
+
>>> info = AutoAction.get_action_info("coding")
|
| 223 |
+
>>> print(info['action_class'])
|
| 224 |
+
'CodingAction'
|
| 225 |
+
>>> print(info['module'])
|
| 226 |
+
'coding_env.client'
|
| 227 |
+
"""
|
| 228 |
+
discovery = get_discovery()
|
| 229 |
+
env_info = discovery.get_environment_by_name(name)
|
| 230 |
+
|
| 231 |
+
if not env_info:
|
| 232 |
+
raise ValueError(f"Unknown environment: {name}")
|
| 233 |
+
|
| 234 |
+
return {
|
| 235 |
+
"env_key": env_info.env_key,
|
| 236 |
+
"env_name": env_info.name,
|
| 237 |
+
"package": env_info.package_name,
|
| 238 |
+
"action_class": env_info.action_class_name,
|
| 239 |
+
"observation_class": env_info.observation_class_name,
|
| 240 |
+
"module": env_info.client_module_path,
|
| 241 |
+
}
|
| 242 |
+
|
| 243 |
+
@classmethod
|
| 244 |
+
def list_actions(cls) -> None:
|
| 245 |
+
"""
|
| 246 |
+
Print a formatted list of all available action classes.
|
| 247 |
+
|
| 248 |
+
This discovers all installed openenv-* packages and displays
|
| 249 |
+
their action class information in a user-friendly format.
|
| 250 |
+
|
| 251 |
+
Examples:
|
| 252 |
+
>>> AutoAction.list_actions()
|
| 253 |
+
Available Action Classes:
|
| 254 |
+
----------------------------------------------------------------------
|
| 255 |
+
echo : EchoAction (from openenv-echo-env)
|
| 256 |
+
coding : CodingAction (from openenv-coding_env)
|
| 257 |
+
----------------------------------------------------------------------
|
| 258 |
+
Total: 2 action classes
|
| 259 |
+
"""
|
| 260 |
+
discovery = get_discovery()
|
| 261 |
+
environments = discovery.discover()
|
| 262 |
+
|
| 263 |
+
print("Available Action Classes:")
|
| 264 |
+
print("-" * 70)
|
| 265 |
+
|
| 266 |
+
if not environments:
|
| 267 |
+
print(" No OpenEnv environments found.")
|
| 268 |
+
print(" Install environments with: pip install openenv-<env-name>")
|
| 269 |
+
else:
|
| 270 |
+
for env_key in sorted(environments.keys()):
|
| 271 |
+
env = environments[env_key]
|
| 272 |
+
print(f" {env_key:<15}: {env.action_class_name}")
|
| 273 |
+
print(f" Package: {env.package_name}")
|
| 274 |
+
|
| 275 |
+
print("-" * 70)
|
| 276 |
+
print(f"Total: {len(environments)} action classes")
|
src/openenv/auto/auto_env.py
ADDED
|
@@ -0,0 +1,896 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Meta Platforms, Inc. and affiliates.
|
| 2 |
+
# All rights reserved.
|
| 3 |
+
#
|
| 4 |
+
# This source code is licensed under the BSD-style license found in the
|
| 5 |
+
# LICENSE file in the root directory of this source tree.
|
| 6 |
+
|
| 7 |
+
"""
|
| 8 |
+
AutoEnv - Automatic Environment Selection
|
| 9 |
+
==========================================
|
| 10 |
+
|
| 11 |
+
AutoEnv provides a HuggingFace-style API for automatically selecting and
|
| 12 |
+
instantiating the correct environment client from installed packages or
|
| 13 |
+
HuggingFace Hub.
|
| 14 |
+
|
| 15 |
+
This module simplifies environment creation by automatically detecting the
|
| 16 |
+
environment type from the name and instantiating the appropriate client class.
|
| 17 |
+
|
| 18 |
+
Example:
|
| 19 |
+
>>> from openenv import AutoEnv, AutoAction
|
| 20 |
+
>>>
|
| 21 |
+
>>> # From installed package
|
| 22 |
+
>>> env = AutoEnv.from_env("coding-env")
|
| 23 |
+
>>>
|
| 24 |
+
>>> # From HuggingFace Hub
|
| 25 |
+
>>> env = AutoEnv.from_env("meta-pytorch/coding-env")
|
| 26 |
+
>>>
|
| 27 |
+
>>> # With configuration
|
| 28 |
+
>>> env = AutoEnv.from_env("coding", env_vars={"DEBUG": "1"})
|
| 29 |
+
"""
|
| 30 |
+
|
| 31 |
+
from __future__ import annotations
|
| 32 |
+
|
| 33 |
+
import importlib
|
| 34 |
+
import logging
|
| 35 |
+
import os
|
| 36 |
+
import shutil
|
| 37 |
+
import subprocess
|
| 38 |
+
import sys
|
| 39 |
+
import requests
|
| 40 |
+
from typing import Any, Optional, TYPE_CHECKING, Dict
|
| 41 |
+
|
| 42 |
+
from ._discovery import get_discovery, _is_hub_url
|
| 43 |
+
from openenv.core.utils import run_async_safely
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
if TYPE_CHECKING:
|
| 47 |
+
from openenv.core.containers.runtime import ContainerProvider
|
| 48 |
+
from openenv.core.env_client import EnvClient
|
| 49 |
+
|
| 50 |
+
logger = logging.getLogger(__name__)
|
| 51 |
+
|
| 52 |
+
# Cache for repo ID → env_name mapping to avoid redundant downloads
|
| 53 |
+
_hub_env_name_cache: Dict[str, str] = {}
|
| 54 |
+
|
| 55 |
+
# Environment variable to skip user confirmation for remote installs
|
| 56 |
+
OPENENV_TRUST_REMOTE_CODE = "OPENENV_TRUST_REMOTE_CODE"
|
| 57 |
+
|
| 58 |
+
|
| 59 |
+
def _has_uv() -> bool:
|
| 60 |
+
"""Check if uv is available in the system."""
|
| 61 |
+
return shutil.which("uv") is not None
|
| 62 |
+
|
| 63 |
+
|
| 64 |
+
def _get_pip_command() -> list[str]:
|
| 65 |
+
"""
|
| 66 |
+
Get the appropriate pip command (uv pip or pip).
|
| 67 |
+
|
| 68 |
+
Returns:
|
| 69 |
+
List of command parts for pip installation
|
| 70 |
+
"""
|
| 71 |
+
if _has_uv():
|
| 72 |
+
return ["uv", "pip"]
|
| 73 |
+
return [sys.executable, "-m", "pip"]
|
| 74 |
+
|
| 75 |
+
|
| 76 |
+
def _confirm_remote_install(repo_id: str) -> bool:
|
| 77 |
+
"""
|
| 78 |
+
Ask user for confirmation before installing remote code.
|
| 79 |
+
|
| 80 |
+
This is a security measure since we're executing code from the internet.
|
| 81 |
+
|
| 82 |
+
Args:
|
| 83 |
+
repo_id: The HuggingFace repo ID being installed
|
| 84 |
+
|
| 85 |
+
Returns:
|
| 86 |
+
True if user confirms, False otherwise
|
| 87 |
+
"""
|
| 88 |
+
# Check environment variable for automated/CI environments
|
| 89 |
+
if os.environ.get(OPENENV_TRUST_REMOTE_CODE, "").lower() in ("1", "true", "yes"):
|
| 90 |
+
logger.info("Skipping confirmation (OPENENV_TRUST_REMOTE_CODE is set)")
|
| 91 |
+
return True
|
| 92 |
+
|
| 93 |
+
# Check if we're in an interactive terminal
|
| 94 |
+
if not sys.stdin.isatty():
|
| 95 |
+
logger.warning(
|
| 96 |
+
"Cannot prompt for confirmation in non-interactive mode. "
|
| 97 |
+
"Set OPENENV_TRUST_REMOTE_CODE=1 to allow remote installs."
|
| 98 |
+
)
|
| 99 |
+
return False
|
| 100 |
+
|
| 101 |
+
print(f"\n{'=' * 60}")
|
| 102 |
+
print("⚠️ SECURITY WARNING: Remote Code Installation")
|
| 103 |
+
print(f"{'=' * 60}")
|
| 104 |
+
print("You are about to install code from a remote repository:")
|
| 105 |
+
print(f" Repository: {repo_id}")
|
| 106 |
+
print(f" Source: https://huggingface.co/spaces/{repo_id}")
|
| 107 |
+
print("\nThis will execute code from the internet on your machine.")
|
| 108 |
+
print("Only proceed if you trust the source.")
|
| 109 |
+
print(f"{'=' * 60}\n")
|
| 110 |
+
|
| 111 |
+
try:
|
| 112 |
+
response = input("Do you want to proceed? [y/N]: ").strip().lower()
|
| 113 |
+
return response in ("y", "yes")
|
| 114 |
+
except (EOFError, KeyboardInterrupt):
|
| 115 |
+
print("\nInstallation cancelled.")
|
| 116 |
+
return False
|
| 117 |
+
|
| 118 |
+
|
| 119 |
+
class AutoEnv:
|
| 120 |
+
"""
|
| 121 |
+
AutoEnv automatically selects and instantiates the correct environment client
|
| 122 |
+
based on environment names or HuggingFace Hub repositories.
|
| 123 |
+
|
| 124 |
+
This class follows the HuggingFace AutoModel pattern, making it easy to work
|
| 125 |
+
with different environments without needing to import specific client classes.
|
| 126 |
+
|
| 127 |
+
The class provides factory methods that:
|
| 128 |
+
1. Check if name is a HuggingFace Hub URL/repo ID
|
| 129 |
+
2. If Hub: download and install the environment package
|
| 130 |
+
3. If local: look up the installed openenv-* package
|
| 131 |
+
4. Import and instantiate the client class
|
| 132 |
+
|
| 133 |
+
Example:
|
| 134 |
+
>>> # From installed package
|
| 135 |
+
>>> env = AutoEnv.from_env("coding-env")
|
| 136 |
+
>>>
|
| 137 |
+
>>> # From HuggingFace Hub
|
| 138 |
+
>>> env = AutoEnv.from_env("meta-pytorch/coding-env")
|
| 139 |
+
>>>
|
| 140 |
+
>>> # List available environments
|
| 141 |
+
>>> AutoEnv.list_environments()
|
| 142 |
+
|
| 143 |
+
Note:
|
| 144 |
+
AutoEnv is not meant to be instantiated directly. Use the class method
|
| 145 |
+
from_env() instead.
|
| 146 |
+
"""
|
| 147 |
+
|
| 148 |
+
def __init__(self):
|
| 149 |
+
"""AutoEnv should not be instantiated directly. Use class methods instead."""
|
| 150 |
+
raise TypeError(
|
| 151 |
+
"AutoEnv is a factory class and should not be instantiated directly. "
|
| 152 |
+
"Use AutoEnv.from_hub() or AutoEnv.from_env() instead."
|
| 153 |
+
)
|
| 154 |
+
|
| 155 |
+
@classmethod
|
| 156 |
+
def _resolve_space_url(cls, repo_id: str) -> str:
|
| 157 |
+
"""
|
| 158 |
+
Resolve HuggingFace Space repo ID to Space URL.
|
| 159 |
+
|
| 160 |
+
Args:
|
| 161 |
+
repo_id: HuggingFace repo ID (e.g., "wukaixingxp/coding-env-test")
|
| 162 |
+
|
| 163 |
+
Returns:
|
| 164 |
+
Space URL (e.g., "https://wukaixingxp-coding-env-test.hf.space")
|
| 165 |
+
|
| 166 |
+
Examples:
|
| 167 |
+
>>> AutoEnv._resolve_space_url("wukaixingxp/coding-env-test")
|
| 168 |
+
'https://wukaixingxp-coding-env-test.hf.space'
|
| 169 |
+
"""
|
| 170 |
+
# Clean up repo_id if it's a full URL
|
| 171 |
+
if "huggingface.co" in repo_id:
|
| 172 |
+
# Extract org/repo from URL
|
| 173 |
+
# https://huggingface.co/wukaixingxp/coding-env-test -> wukaixingxp/coding-env-test
|
| 174 |
+
parts = repo_id.split("/")
|
| 175 |
+
if len(parts) >= 2:
|
| 176 |
+
repo_id = f"{parts[-2]}/{parts[-1]}"
|
| 177 |
+
|
| 178 |
+
# Convert user/space-name to user-space-name.hf.space
|
| 179 |
+
space_slug = repo_id.replace("/", "-")
|
| 180 |
+
return f"https://{space_slug}.hf.space"
|
| 181 |
+
|
| 182 |
+
@classmethod
|
| 183 |
+
def _is_local_url(cls, url: str) -> bool:
|
| 184 |
+
"""
|
| 185 |
+
Check if a URL points to a local server.
|
| 186 |
+
|
| 187 |
+
Args:
|
| 188 |
+
url: URL to check
|
| 189 |
+
|
| 190 |
+
Returns:
|
| 191 |
+
True if URL is localhost or 127.0.0.1, False otherwise
|
| 192 |
+
|
| 193 |
+
Examples:
|
| 194 |
+
>>> AutoEnv._is_local_url("http://localhost:8000")
|
| 195 |
+
True
|
| 196 |
+
>>> AutoEnv._is_local_url("http://127.0.0.1:8000")
|
| 197 |
+
True
|
| 198 |
+
>>> AutoEnv._is_local_url("https://example.com")
|
| 199 |
+
False
|
| 200 |
+
"""
|
| 201 |
+
url_lower = url.lower()
|
| 202 |
+
return "localhost" in url_lower or "127.0.0.1" in url_lower
|
| 203 |
+
|
| 204 |
+
@classmethod
|
| 205 |
+
def _check_server_availability(cls, base_url: str, timeout: float = 2.0) -> bool:
|
| 206 |
+
"""
|
| 207 |
+
Check if a server at the given URL is running and accessible.
|
| 208 |
+
|
| 209 |
+
Args:
|
| 210 |
+
base_url: Server base URL to check
|
| 211 |
+
timeout: Request timeout in seconds
|
| 212 |
+
|
| 213 |
+
Returns:
|
| 214 |
+
True if server is accessible, False otherwise
|
| 215 |
+
|
| 216 |
+
Examples:
|
| 217 |
+
>>> AutoEnv._check_server_availability("http://localhost:8000")
|
| 218 |
+
True # if server is running
|
| 219 |
+
"""
|
| 220 |
+
try:
|
| 221 |
+
# Bypass proxy for localhost to avoid proxy issues
|
| 222 |
+
proxies = None
|
| 223 |
+
if cls._is_local_url(base_url):
|
| 224 |
+
proxies = {"http": None, "https": None}
|
| 225 |
+
|
| 226 |
+
# Try to access the health endpoint
|
| 227 |
+
response = requests.get(
|
| 228 |
+
f"{base_url}/health", timeout=timeout, proxies=proxies
|
| 229 |
+
)
|
| 230 |
+
if response.status_code == 200:
|
| 231 |
+
return True
|
| 232 |
+
|
| 233 |
+
# If health endpoint doesn't exist, try root endpoint
|
| 234 |
+
response = requests.get(base_url, timeout=timeout, proxies=proxies)
|
| 235 |
+
return response.status_code == 200
|
| 236 |
+
except (requests.RequestException, Exception) as e:
|
| 237 |
+
logger.debug(f"Server {base_url} not accessible: {e}")
|
| 238 |
+
return False
|
| 239 |
+
|
| 240 |
+
@classmethod
|
| 241 |
+
def _check_space_availability(cls, space_url: str, timeout: float = 5.0) -> bool:
|
| 242 |
+
"""
|
| 243 |
+
Check if HuggingFace Space is running and accessible.
|
| 244 |
+
|
| 245 |
+
Args:
|
| 246 |
+
space_url: Space URL to check
|
| 247 |
+
timeout: Request timeout in seconds
|
| 248 |
+
|
| 249 |
+
Returns:
|
| 250 |
+
True if Space is accessible, False otherwise
|
| 251 |
+
|
| 252 |
+
Examples:
|
| 253 |
+
>>> AutoEnv._check_space_availability("https://wukaixingxp-coding-env-test.hf.space")
|
| 254 |
+
True
|
| 255 |
+
"""
|
| 256 |
+
try:
|
| 257 |
+
# Try to access the health endpoint
|
| 258 |
+
response = requests.get(f"{space_url}/health", timeout=timeout)
|
| 259 |
+
if response.status_code == 200:
|
| 260 |
+
return True
|
| 261 |
+
|
| 262 |
+
# If health endpoint doesn't exist, try root endpoint
|
| 263 |
+
response = requests.get(space_url, timeout=timeout)
|
| 264 |
+
return response.status_code == 200
|
| 265 |
+
except (requests.RequestException, Exception) as e:
|
| 266 |
+
logger.debug(f"Space {space_url} not accessible: {e}")
|
| 267 |
+
return False
|
| 268 |
+
|
| 269 |
+
@classmethod
|
| 270 |
+
def _get_hub_git_url(cls, repo_id: str) -> str:
|
| 271 |
+
"""
|
| 272 |
+
Get the git URL for a HuggingFace Space.
|
| 273 |
+
|
| 274 |
+
Args:
|
| 275 |
+
repo_id: HuggingFace repo ID (e.g., "wukaixingxp/coding-env-test")
|
| 276 |
+
|
| 277 |
+
Returns:
|
| 278 |
+
Git URL for pip installation (e.g., "git+https://huggingface.co/spaces/wukaixingxp/coding-env-test")
|
| 279 |
+
"""
|
| 280 |
+
# Clean up repo_id if it's a full URL
|
| 281 |
+
if "huggingface.co" in repo_id:
|
| 282 |
+
parts = repo_id.split("/")
|
| 283 |
+
if len(parts) >= 2:
|
| 284 |
+
repo_id = f"{parts[-2]}/{parts[-1]}"
|
| 285 |
+
|
| 286 |
+
return f"git+https://huggingface.co/spaces/{repo_id}"
|
| 287 |
+
|
| 288 |
+
@classmethod
|
| 289 |
+
def _install_from_hub(cls, repo_id: str, trust_remote_code: bool = False) -> str:
|
| 290 |
+
"""
|
| 291 |
+
Install environment package directly from HuggingFace Hub using git+.
|
| 292 |
+
|
| 293 |
+
This is the preferred method as it avoids downloading the entire repo
|
| 294 |
+
and uses pip/uv's native git support.
|
| 295 |
+
|
| 296 |
+
Args:
|
| 297 |
+
repo_id: HuggingFace repo ID (e.g., "wukaixingxp/coding-env-test")
|
| 298 |
+
trust_remote_code: If True, skip user confirmation
|
| 299 |
+
|
| 300 |
+
Returns:
|
| 301 |
+
Package name that was installed
|
| 302 |
+
|
| 303 |
+
Raises:
|
| 304 |
+
ValueError: If installation fails or user declines
|
| 305 |
+
"""
|
| 306 |
+
# Security check - confirm with user before installing remote code
|
| 307 |
+
if not trust_remote_code and not _confirm_remote_install(repo_id):
|
| 308 |
+
raise ValueError(
|
| 309 |
+
"Installation cancelled by user.\n"
|
| 310 |
+
"To allow remote installs without prompting, set OPENENV_TRUST_REMOTE_CODE=1"
|
| 311 |
+
)
|
| 312 |
+
|
| 313 |
+
git_url = cls._get_hub_git_url(repo_id)
|
| 314 |
+
pip_cmd = _get_pip_command()
|
| 315 |
+
pip_name = "uv pip" if pip_cmd[0] == "uv" else "pip"
|
| 316 |
+
|
| 317 |
+
logger.info(f"Installing from HuggingFace Space using {pip_name}: {repo_id}")
|
| 318 |
+
logger.info(f"Command: {' '.join(pip_cmd)} install {git_url}")
|
| 319 |
+
|
| 320 |
+
try:
|
| 321 |
+
result = subprocess.run(
|
| 322 |
+
[*pip_cmd, "install", git_url],
|
| 323 |
+
check=True,
|
| 324 |
+
capture_output=True,
|
| 325 |
+
text=True,
|
| 326 |
+
)
|
| 327 |
+
|
| 328 |
+
# Try to extract package name from pip output
|
| 329 |
+
# Look for "Successfully installed <package-name>-<version>"
|
| 330 |
+
for line in result.stdout.split("\n"):
|
| 331 |
+
if "Successfully installed" in line:
|
| 332 |
+
# Parse package name from the line
|
| 333 |
+
parts = line.replace("Successfully installed", "").strip().split()
|
| 334 |
+
for part in parts:
|
| 335 |
+
if part.startswith("openenv-"):
|
| 336 |
+
# Remove version suffix (e.g., "openenv-coding_env-0.1.0" -> "openenv-coding_env")
|
| 337 |
+
# Check if last segment looks like a version number
|
| 338 |
+
last_segment = part.rsplit("-", 1)[-1]
|
| 339 |
+
if last_segment.replace(".", "").isdigit():
|
| 340 |
+
package_name = "-".join(part.rsplit("-", 1)[:-1])
|
| 341 |
+
else:
|
| 342 |
+
package_name = part
|
| 343 |
+
logger.info(f"Successfully installed: {package_name}")
|
| 344 |
+
return package_name
|
| 345 |
+
|
| 346 |
+
# Fallback: try to determine package name from repo_id
|
| 347 |
+
# Convention: repo name like "coding-env-test" -> package "openenv-coding_env"
|
| 348 |
+
env_name = repo_id.split("/")[-1] # Get repo name from "user/repo"
|
| 349 |
+
env_name = env_name.replace("-", "_")
|
| 350 |
+
if not env_name.endswith("_env"):
|
| 351 |
+
env_name = f"{env_name}_env"
|
| 352 |
+
package_name = f"openenv-{env_name}"
|
| 353 |
+
|
| 354 |
+
logger.info(f"Installed (inferred package name): {package_name}")
|
| 355 |
+
return package_name
|
| 356 |
+
|
| 357 |
+
except subprocess.CalledProcessError as e:
|
| 358 |
+
error_msg = e.stderr or e.stdout or str(e)
|
| 359 |
+
raise ValueError(
|
| 360 |
+
f"Failed to install environment from HuggingFace Space: {repo_id}\n"
|
| 361 |
+
f"Command: {' '.join(pip_cmd)} install {git_url}\n"
|
| 362 |
+
f"Error: {error_msg}\n"
|
| 363 |
+
f"Make sure the repository exists and contains a valid Python package."
|
| 364 |
+
) from e
|
| 365 |
+
|
| 366 |
+
@classmethod
|
| 367 |
+
def _is_package_installed(cls, package_name: str) -> bool:
|
| 368 |
+
"""
|
| 369 |
+
Check if a package is already installed.
|
| 370 |
+
|
| 371 |
+
Args:
|
| 372 |
+
package_name: Package name (e.g., "openenv-coding_env")
|
| 373 |
+
|
| 374 |
+
Returns:
|
| 375 |
+
True if installed, False otherwise
|
| 376 |
+
"""
|
| 377 |
+
try:
|
| 378 |
+
import importlib.metadata
|
| 379 |
+
|
| 380 |
+
importlib.metadata.distribution(package_name)
|
| 381 |
+
return True
|
| 382 |
+
except importlib.metadata.PackageNotFoundError:
|
| 383 |
+
return False
|
| 384 |
+
|
| 385 |
+
@classmethod
|
| 386 |
+
def _ensure_package_from_hub(
|
| 387 |
+
cls, name: str, trust_remote_code: bool = False
|
| 388 |
+
) -> str:
|
| 389 |
+
"""
|
| 390 |
+
Ensure package from HuggingFace Hub is installed.
|
| 391 |
+
|
| 392 |
+
Uses git+ URLs for direct installation without downloading the entire repo.
|
| 393 |
+
Prompts user for confirmation before installing remote code.
|
| 394 |
+
|
| 395 |
+
Args:
|
| 396 |
+
name: HuggingFace repo ID (e.g., "wukaixingxp/coding-env-test")
|
| 397 |
+
trust_remote_code: If True, skip user confirmation
|
| 398 |
+
|
| 399 |
+
Returns:
|
| 400 |
+
Environment name (e.g., "coding_env")
|
| 401 |
+
"""
|
| 402 |
+
global _hub_env_name_cache
|
| 403 |
+
|
| 404 |
+
# Check if we already resolved this repo ID
|
| 405 |
+
if name in _hub_env_name_cache:
|
| 406 |
+
env_name = _hub_env_name_cache[name]
|
| 407 |
+
logger.debug(f"Using cached env name for {name}: {env_name}")
|
| 408 |
+
return env_name
|
| 409 |
+
|
| 410 |
+
# Try to infer expected package name from repo ID
|
| 411 |
+
# Convention: repo "user/coding-env" -> package "openenv-coding_env"
|
| 412 |
+
repo_name = name.split("/")[-1] if "/" in name else name
|
| 413 |
+
expected_env_name = repo_name.replace("-", "_")
|
| 414 |
+
if not expected_env_name.endswith("_env"):
|
| 415 |
+
expected_env_name = f"{expected_env_name}_env"
|
| 416 |
+
expected_package_name = f"openenv-{expected_env_name}"
|
| 417 |
+
|
| 418 |
+
# Check if already installed
|
| 419 |
+
if cls._is_package_installed(expected_package_name):
|
| 420 |
+
logger.info(f"Package already installed: {expected_package_name}")
|
| 421 |
+
# Clear and refresh discovery cache to make sure it's detected
|
| 422 |
+
get_discovery().clear_cache()
|
| 423 |
+
get_discovery().discover(use_cache=False)
|
| 424 |
+
# Cache the result
|
| 425 |
+
_hub_env_name_cache[name] = expected_env_name
|
| 426 |
+
return expected_env_name
|
| 427 |
+
|
| 428 |
+
# Not installed, install using git+ URL
|
| 429 |
+
logger.info(f"Package not found locally, installing from Hub: {name}")
|
| 430 |
+
|
| 431 |
+
# Track existing packages before installation
|
| 432 |
+
get_discovery().clear_cache()
|
| 433 |
+
existing_envs = set(get_discovery().discover(use_cache=False).keys())
|
| 434 |
+
|
| 435 |
+
# Install the package
|
| 436 |
+
cls._install_from_hub(name, trust_remote_code=trust_remote_code)
|
| 437 |
+
|
| 438 |
+
# Clear discovery cache to pick up the newly installed package
|
| 439 |
+
try:
|
| 440 |
+
importlib.invalidate_caches()
|
| 441 |
+
except Exception:
|
| 442 |
+
pass
|
| 443 |
+
get_discovery().clear_cache()
|
| 444 |
+
discovered_envs = get_discovery().discover(use_cache=False)
|
| 445 |
+
|
| 446 |
+
# Find the newly installed environment by comparing before/after
|
| 447 |
+
new_envs = set(discovered_envs.keys()) - existing_envs
|
| 448 |
+
|
| 449 |
+
if new_envs:
|
| 450 |
+
# Use the first newly discovered environment
|
| 451 |
+
env_name = next(iter(new_envs))
|
| 452 |
+
logger.info(f"Found newly installed environment: '{env_name}'")
|
| 453 |
+
else:
|
| 454 |
+
# Fallback: try to find by matching module patterns
|
| 455 |
+
# Look for any env that might match the repo name pattern
|
| 456 |
+
repo_name = name.split("/")[-1] if "/" in name else name
|
| 457 |
+
repo_base = (
|
| 458 |
+
repo_name.replace("-", "_").replace("_env", "").replace("_test", "")
|
| 459 |
+
)
|
| 460 |
+
|
| 461 |
+
env_name = None
|
| 462 |
+
for env_key, env_info in discovered_envs.items():
|
| 463 |
+
# Check if env_key is a prefix/substring match
|
| 464 |
+
if env_key in repo_base or repo_base.startswith(env_key):
|
| 465 |
+
env_name = env_key
|
| 466 |
+
logger.info(
|
| 467 |
+
f"Found matching environment '{env_name}' for repo '{name}'"
|
| 468 |
+
)
|
| 469 |
+
break
|
| 470 |
+
|
| 471 |
+
if env_name is None:
|
| 472 |
+
# Last resort: use inferred name from repo
|
| 473 |
+
env_name = repo_name.replace("-", "_")
|
| 474 |
+
if not env_name.endswith("_env"):
|
| 475 |
+
env_name = f"{env_name}_env"
|
| 476 |
+
# Strip to get env_key
|
| 477 |
+
env_name = env_name.replace("_env", "")
|
| 478 |
+
logger.warning(
|
| 479 |
+
f"Could not find newly installed environment for repo '{name}', "
|
| 480 |
+
f"using inferred name: {env_name}"
|
| 481 |
+
)
|
| 482 |
+
|
| 483 |
+
# Cache the result to avoid redundant installs
|
| 484 |
+
_hub_env_name_cache[name] = env_name
|
| 485 |
+
|
| 486 |
+
return env_name
|
| 487 |
+
|
| 488 |
+
@classmethod
|
| 489 |
+
def from_env(
|
| 490 |
+
cls,
|
| 491 |
+
name: str,
|
| 492 |
+
base_url: Optional[str] = None,
|
| 493 |
+
docker_image: Optional[str] = None,
|
| 494 |
+
container_provider: Optional[ContainerProvider] = None,
|
| 495 |
+
wait_timeout: float = 30.0,
|
| 496 |
+
env_vars: Optional[Dict[str, str]] = None,
|
| 497 |
+
trust_remote_code: bool = False,
|
| 498 |
+
skip_install: bool = False,
|
| 499 |
+
**kwargs: Any,
|
| 500 |
+
) -> "EnvClient":
|
| 501 |
+
"""
|
| 502 |
+
Create an environment client from a name or HuggingFace Hub repository.
|
| 503 |
+
|
| 504 |
+
This method automatically:
|
| 505 |
+
1. Checks if the name is a HuggingFace Hub URL/repo ID
|
| 506 |
+
2. If Hub: installs the environment package using git+ URL
|
| 507 |
+
3. If local: looks up the installed openenv-* package
|
| 508 |
+
4. Imports the client class and instantiates it
|
| 509 |
+
|
| 510 |
+
Args:
|
| 511 |
+
name: Environment name or HuggingFace Hub repo ID
|
| 512 |
+
Examples:
|
| 513 |
+
- "coding" / "coding-env" / "coding_env"
|
| 514 |
+
- "meta-pytorch/coding-env" (Hub repo ID)
|
| 515 |
+
- "https://huggingface.co/meta-pytorch/coding-env" (Hub URL)
|
| 516 |
+
base_url: Optional base URL for HTTP connection
|
| 517 |
+
docker_image: Optional Docker image name (overrides default)
|
| 518 |
+
container_provider: Optional container provider
|
| 519 |
+
wait_timeout: Timeout for container startup (seconds)
|
| 520 |
+
env_vars: Optional environment variables for the container
|
| 521 |
+
trust_remote_code: If True, skip user confirmation when installing
|
| 522 |
+
from HuggingFace Hub. Can also be set via OPENENV_TRUST_REMOTE_CODE
|
| 523 |
+
environment variable.
|
| 524 |
+
skip_install: If True, skip package installation and return a
|
| 525 |
+
GenericEnvClient for remote environments. Useful when you only
|
| 526 |
+
want to connect to a running server without installing any
|
| 527 |
+
remote code. When True:
|
| 528 |
+
- If base_url is provided: connects directly using GenericEnvClient
|
| 529 |
+
- If HF Space is running: connects to Space using GenericEnvClient
|
| 530 |
+
- If HF Space is not running: uses Docker from HF registry
|
| 531 |
+
**kwargs: Additional arguments passed to the client class
|
| 532 |
+
|
| 533 |
+
Returns:
|
| 534 |
+
Instance of the environment client class
|
| 535 |
+
|
| 536 |
+
Raises:
|
| 537 |
+
ValueError: If environment not found or cannot be loaded
|
| 538 |
+
ImportError: If environment package is not installed
|
| 539 |
+
|
| 540 |
+
Examples:
|
| 541 |
+
>>> # From installed package
|
| 542 |
+
>>> env = AutoEnv.from_env("coding-env")
|
| 543 |
+
>>>
|
| 544 |
+
>>> # From HuggingFace Hub
|
| 545 |
+
>>> env = AutoEnv.from_env("meta-pytorch/coding-env")
|
| 546 |
+
>>>
|
| 547 |
+
>>> # With custom Docker image
|
| 548 |
+
>>> env = AutoEnv.from_env("coding", docker_image="my-coding-env:v2")
|
| 549 |
+
>>>
|
| 550 |
+
>>> # With environment variables
|
| 551 |
+
>>> env = AutoEnv.from_env(
|
| 552 |
+
... "dipg",
|
| 553 |
+
... env_vars={"DIPG_DATASET_PATH": "/data/dipg"}
|
| 554 |
+
... )
|
| 555 |
+
>>>
|
| 556 |
+
>>> # Skip package installation, use GenericEnvClient
|
| 557 |
+
>>> env = AutoEnv.from_env(
|
| 558 |
+
... "user/my-env",
|
| 559 |
+
... skip_install=True
|
| 560 |
+
... )
|
| 561 |
+
"""
|
| 562 |
+
from openenv.core import GenericEnvClient
|
| 563 |
+
|
| 564 |
+
# Handle skip_install mode - return GenericEnvClient without package installation
|
| 565 |
+
if skip_install:
|
| 566 |
+
# If base_url is provided, connect directly
|
| 567 |
+
if base_url:
|
| 568 |
+
if cls._check_server_availability(base_url):
|
| 569 |
+
logger.info(
|
| 570 |
+
f"Using GenericEnvClient for {base_url} (skip_install=True)"
|
| 571 |
+
)
|
| 572 |
+
return GenericEnvClient(base_url=base_url, **kwargs)
|
| 573 |
+
else:
|
| 574 |
+
raise ConnectionError(
|
| 575 |
+
f"Server not available at {base_url}. "
|
| 576 |
+
f"Please ensure the server is running."
|
| 577 |
+
)
|
| 578 |
+
|
| 579 |
+
# If it's a Hub URL, try to connect to Space or use Docker
|
| 580 |
+
if _is_hub_url(name):
|
| 581 |
+
space_url = cls._resolve_space_url(name)
|
| 582 |
+
logger.info(f"Checking if HuggingFace Space is accessible: {space_url}")
|
| 583 |
+
|
| 584 |
+
if cls._check_space_availability(space_url):
|
| 585 |
+
logger.info(
|
| 586 |
+
f"Using GenericEnvClient for Space {space_url} (skip_install=True)"
|
| 587 |
+
)
|
| 588 |
+
return GenericEnvClient(base_url=space_url, **kwargs)
|
| 589 |
+
else:
|
| 590 |
+
# Space not running, use Docker from HF registry
|
| 591 |
+
logger.info(
|
| 592 |
+
f"Space not running at {space_url}, "
|
| 593 |
+
f"using GenericEnvClient with HF Docker registry"
|
| 594 |
+
)
|
| 595 |
+
return run_async_safely(
|
| 596 |
+
GenericEnvClient.from_env(
|
| 597 |
+
name,
|
| 598 |
+
use_docker=True,
|
| 599 |
+
provider=container_provider,
|
| 600 |
+
env_vars=env_vars or {},
|
| 601 |
+
**kwargs,
|
| 602 |
+
)
|
| 603 |
+
)
|
| 604 |
+
|
| 605 |
+
# For local environments with skip_install, we need docker_image
|
| 606 |
+
if docker_image:
|
| 607 |
+
logger.info(
|
| 608 |
+
f"Using GenericEnvClient with Docker image {docker_image} "
|
| 609 |
+
f"(skip_install=True)"
|
| 610 |
+
)
|
| 611 |
+
return run_async_safely(
|
| 612 |
+
GenericEnvClient.from_docker_image(
|
| 613 |
+
image=docker_image,
|
| 614 |
+
provider=container_provider,
|
| 615 |
+
wait_timeout=wait_timeout,
|
| 616 |
+
env_vars=env_vars or {},
|
| 617 |
+
**kwargs,
|
| 618 |
+
)
|
| 619 |
+
)
|
| 620 |
+
else:
|
| 621 |
+
raise ValueError(
|
| 622 |
+
f"Cannot use skip_install=True for local environment '{name}' "
|
| 623 |
+
f"without providing base_url or docker_image. "
|
| 624 |
+
f"For local environments, either:\n"
|
| 625 |
+
f" 1. Provide base_url to connect to a running server\n"
|
| 626 |
+
f" 2. Provide docker_image to start a container\n"
|
| 627 |
+
f" 3. Set skip_install=False to use the installed package"
|
| 628 |
+
)
|
| 629 |
+
|
| 630 |
+
# Check if it's a HuggingFace Hub URL or repo ID
|
| 631 |
+
if _is_hub_url(name):
|
| 632 |
+
# Try to connect to Space directly first
|
| 633 |
+
space_url = cls._resolve_space_url(name)
|
| 634 |
+
logger.info(f"Checking if HuggingFace Space is accessible: {space_url}")
|
| 635 |
+
|
| 636 |
+
space_is_available = cls._check_space_availability(space_url)
|
| 637 |
+
|
| 638 |
+
if space_is_available and base_url is None:
|
| 639 |
+
# Space is accessible! We'll connect directly without Docker
|
| 640 |
+
logger.info(f"Space is accessible at: {space_url}")
|
| 641 |
+
logger.info("Installing package for client code (no Docker needed)...")
|
| 642 |
+
|
| 643 |
+
# Ensure package is installed (uses git+ URL)
|
| 644 |
+
env_name = cls._ensure_package_from_hub(
|
| 645 |
+
name, trust_remote_code=trust_remote_code
|
| 646 |
+
)
|
| 647 |
+
|
| 648 |
+
# Set base_url to connect to remote Space
|
| 649 |
+
base_url = space_url
|
| 650 |
+
logger.info("Will connect to remote Space (no local Docker)")
|
| 651 |
+
else:
|
| 652 |
+
# Space not accessible or user provided explicit base_url
|
| 653 |
+
if not space_is_available:
|
| 654 |
+
logger.info(f"Space not accessible at {space_url}")
|
| 655 |
+
logger.info("Falling back to local Docker mode...")
|
| 656 |
+
|
| 657 |
+
# Ensure package is installed (uses git+ URL)
|
| 658 |
+
env_name = cls._ensure_package_from_hub(
|
| 659 |
+
name, trust_remote_code=trust_remote_code
|
| 660 |
+
)
|
| 661 |
+
else:
|
| 662 |
+
env_name = name
|
| 663 |
+
|
| 664 |
+
# Get environment info from discovery
|
| 665 |
+
discovery = get_discovery()
|
| 666 |
+
env_info = discovery.get_environment_by_name(env_name)
|
| 667 |
+
|
| 668 |
+
if not env_info:
|
| 669 |
+
# Environment not found - provide helpful error message
|
| 670 |
+
available_envs = discovery.discover()
|
| 671 |
+
|
| 672 |
+
if not available_envs:
|
| 673 |
+
raise ValueError(
|
| 674 |
+
"No OpenEnv environments found.\n"
|
| 675 |
+
"Install an environment with: pip install openenv-<env-name>\n"
|
| 676 |
+
"Or specify a HuggingFace Hub repository: AutoEnv.from_env('openenv/echo_env')"
|
| 677 |
+
)
|
| 678 |
+
|
| 679 |
+
# Try to suggest similar environment names
|
| 680 |
+
from difflib import get_close_matches
|
| 681 |
+
|
| 682 |
+
env_keys = list(available_envs.keys())
|
| 683 |
+
suggestions = get_close_matches(env_name, env_keys, n=3, cutoff=0.6)
|
| 684 |
+
|
| 685 |
+
error_msg = f"Unknown environment '{env_name}'.\n"
|
| 686 |
+
if suggestions:
|
| 687 |
+
error_msg += f"Did you mean: {', '.join(suggestions)}?\n"
|
| 688 |
+
error_msg += f"Available environments: {', '.join(sorted(env_keys))}"
|
| 689 |
+
|
| 690 |
+
raise ValueError(error_msg)
|
| 691 |
+
|
| 692 |
+
# Get the client class
|
| 693 |
+
try:
|
| 694 |
+
client_class = env_info.get_client_class()
|
| 695 |
+
except ImportError as e:
|
| 696 |
+
raise ImportError(
|
| 697 |
+
f"Failed to import environment client for '{env_name}'.\n"
|
| 698 |
+
f"Package '{env_info.package_name}' appears to be installed but the module cannot be imported.\n"
|
| 699 |
+
f"Try reinstalling: pip install --force-reinstall {env_info.package_name}\n"
|
| 700 |
+
f"Original error: {e}"
|
| 701 |
+
) from e
|
| 702 |
+
|
| 703 |
+
# Determine Docker image to use
|
| 704 |
+
if docker_image is None:
|
| 705 |
+
docker_image = env_info.default_image
|
| 706 |
+
|
| 707 |
+
# Create client instance
|
| 708 |
+
try:
|
| 709 |
+
if base_url:
|
| 710 |
+
# Check if the server at base_url is available
|
| 711 |
+
is_local = cls._is_local_url(base_url)
|
| 712 |
+
server_available = cls._check_server_availability(base_url)
|
| 713 |
+
|
| 714 |
+
if server_available:
|
| 715 |
+
# Server is running, connect directly
|
| 716 |
+
logger.info(
|
| 717 |
+
f"✅ Server available at {base_url}, connecting directly"
|
| 718 |
+
)
|
| 719 |
+
return client_class(base_url=base_url, provider=None, **kwargs)
|
| 720 |
+
elif is_local:
|
| 721 |
+
# Local server not running, auto-start Docker container
|
| 722 |
+
logger.info(f"❌ Server not available at {base_url}")
|
| 723 |
+
logger.info(f"🐳 Auto-starting Docker container: {docker_image}")
|
| 724 |
+
return run_async_safely(
|
| 725 |
+
client_class.from_docker_image(
|
| 726 |
+
image=docker_image,
|
| 727 |
+
provider=container_provider,
|
| 728 |
+
wait_timeout=wait_timeout,
|
| 729 |
+
env_vars=env_vars or {},
|
| 730 |
+
**kwargs,
|
| 731 |
+
)
|
| 732 |
+
)
|
| 733 |
+
else:
|
| 734 |
+
# Remote server not available, cannot auto-start
|
| 735 |
+
raise ConnectionError(
|
| 736 |
+
f"Remote server not available at {base_url}. "
|
| 737 |
+
f"Please ensure the server is running."
|
| 738 |
+
)
|
| 739 |
+
else:
|
| 740 |
+
# No base_url provided, start new Docker container
|
| 741 |
+
return run_async_safely(
|
| 742 |
+
client_class.from_docker_image(
|
| 743 |
+
image=docker_image,
|
| 744 |
+
provider=container_provider,
|
| 745 |
+
wait_timeout=wait_timeout,
|
| 746 |
+
env_vars=env_vars or {},
|
| 747 |
+
**kwargs,
|
| 748 |
+
)
|
| 749 |
+
)
|
| 750 |
+
except Exception as e:
|
| 751 |
+
raise ValueError(
|
| 752 |
+
f"Failed to create environment client for '{env_name}'.\n"
|
| 753 |
+
f"Client class: {client_class.__name__}\n"
|
| 754 |
+
f"Docker image: {docker_image}\n"
|
| 755 |
+
f"Error: {e}"
|
| 756 |
+
) from e
|
| 757 |
+
|
| 758 |
+
@classmethod
|
| 759 |
+
def from_hub(
|
| 760 |
+
cls,
|
| 761 |
+
name: str,
|
| 762 |
+
base_url: Optional[str] = None,
|
| 763 |
+
docker_image: Optional[str] = None,
|
| 764 |
+
container_provider: Optional["ContainerProvider"] = None,
|
| 765 |
+
wait_timeout: float = 30.0,
|
| 766 |
+
env_vars: Optional[Dict[str, str]] = None,
|
| 767 |
+
trust_remote_code: bool = False,
|
| 768 |
+
skip_install: bool = False,
|
| 769 |
+
**kwargs: Any,
|
| 770 |
+
) -> "EnvClient":
|
| 771 |
+
"""
|
| 772 |
+
Create an environment client from a name or HuggingFace Hub repository.
|
| 773 |
+
|
| 774 |
+
This is an alias for from_env() for backward compatibility.
|
| 775 |
+
|
| 776 |
+
Args:
|
| 777 |
+
name: Environment name or HuggingFace Hub repo ID
|
| 778 |
+
base_url: Optional base URL for HTTP connection
|
| 779 |
+
docker_image: Optional Docker image name (overrides default)
|
| 780 |
+
container_provider: Optional container provider
|
| 781 |
+
wait_timeout: Timeout for container startup (seconds)
|
| 782 |
+
env_vars: Optional environment variables for the container
|
| 783 |
+
trust_remote_code: If True, skip user confirmation when installing
|
| 784 |
+
from HuggingFace Hub
|
| 785 |
+
skip_install: If True, skip package installation and return a
|
| 786 |
+
GenericEnvClient for remote environments
|
| 787 |
+
**kwargs: Additional arguments passed to the client class
|
| 788 |
+
|
| 789 |
+
Returns:
|
| 790 |
+
Instance of the environment client class
|
| 791 |
+
|
| 792 |
+
Examples:
|
| 793 |
+
>>> env = AutoEnv.from_hub("coding-env")
|
| 794 |
+
>>> env = AutoEnv.from_hub("meta-pytorch/coding-env")
|
| 795 |
+
"""
|
| 796 |
+
return cls.from_env(
|
| 797 |
+
name=name,
|
| 798 |
+
base_url=base_url,
|
| 799 |
+
docker_image=docker_image,
|
| 800 |
+
container_provider=container_provider,
|
| 801 |
+
wait_timeout=wait_timeout,
|
| 802 |
+
env_vars=env_vars,
|
| 803 |
+
trust_remote_code=trust_remote_code,
|
| 804 |
+
skip_install=skip_install,
|
| 805 |
+
**kwargs,
|
| 806 |
+
)
|
| 807 |
+
|
| 808 |
+
@classmethod
|
| 809 |
+
def get_env_class(cls, name: str):
|
| 810 |
+
"""
|
| 811 |
+
Get the environment client class without instantiating it.
|
| 812 |
+
|
| 813 |
+
Args:
|
| 814 |
+
name: Environment name
|
| 815 |
+
|
| 816 |
+
Returns:
|
| 817 |
+
The environment client class
|
| 818 |
+
|
| 819 |
+
Raises:
|
| 820 |
+
ValueError: If environment not found
|
| 821 |
+
|
| 822 |
+
Examples:
|
| 823 |
+
>>> CodingEnv = AutoEnv.get_env_class("coding")
|
| 824 |
+
>>> # Now you can instantiate it yourself
|
| 825 |
+
>>> env = CodingEnv(base_url="http://localhost:8000")
|
| 826 |
+
"""
|
| 827 |
+
discovery = get_discovery()
|
| 828 |
+
env_info = discovery.get_environment_by_name(name)
|
| 829 |
+
|
| 830 |
+
if not env_info:
|
| 831 |
+
raise ValueError(f"Unknown environment: {name}")
|
| 832 |
+
|
| 833 |
+
return env_info.get_client_class()
|
| 834 |
+
|
| 835 |
+
@classmethod
|
| 836 |
+
def get_env_info(cls, name: str) -> Dict[str, Any]:
|
| 837 |
+
"""
|
| 838 |
+
Get detailed information about an environment.
|
| 839 |
+
|
| 840 |
+
Args:
|
| 841 |
+
name: Environment name
|
| 842 |
+
|
| 843 |
+
Returns:
|
| 844 |
+
Dictionary with environment metadata
|
| 845 |
+
|
| 846 |
+
Raises:
|
| 847 |
+
ValueError: If environment not found
|
| 848 |
+
|
| 849 |
+
Examples:
|
| 850 |
+
>>> info = AutoEnv.get_env_info("coding")
|
| 851 |
+
>>> print(info['description'])
|
| 852 |
+
'Coding environment for OpenEnv'
|
| 853 |
+
>>> print(info['default_image'])
|
| 854 |
+
'coding-env:latest'
|
| 855 |
+
"""
|
| 856 |
+
discovery = get_discovery()
|
| 857 |
+
env_info = discovery.get_environment_by_name(name)
|
| 858 |
+
|
| 859 |
+
if not env_info:
|
| 860 |
+
raise ValueError(f"Unknown environment: {name}")
|
| 861 |
+
|
| 862 |
+
return {
|
| 863 |
+
"env_key": env_info.env_key,
|
| 864 |
+
"name": env_info.name,
|
| 865 |
+
"package": env_info.package_name,
|
| 866 |
+
"version": env_info.version,
|
| 867 |
+
"description": env_info.description,
|
| 868 |
+
"env_class": env_info.client_class_name,
|
| 869 |
+
"action_class": env_info.action_class_name,
|
| 870 |
+
"observation_class": env_info.observation_class_name,
|
| 871 |
+
"module": env_info.client_module_path,
|
| 872 |
+
"default_image": env_info.default_image,
|
| 873 |
+
"spec_version": env_info.spec_version,
|
| 874 |
+
}
|
| 875 |
+
|
| 876 |
+
@classmethod
|
| 877 |
+
def list_environments(cls) -> None:
|
| 878 |
+
"""
|
| 879 |
+
Print a formatted list of all available environments.
|
| 880 |
+
|
| 881 |
+
This discovers all installed openenv-* packages and displays
|
| 882 |
+
their metadata in a user-friendly format.
|
| 883 |
+
|
| 884 |
+
Examples:
|
| 885 |
+
>>> AutoEnv.list_environments()
|
| 886 |
+
Available OpenEnv Environments:
|
| 887 |
+
----------------------------------------------------------------------
|
| 888 |
+
echo : Echo Environment (v0.1.0)
|
| 889 |
+
Package: openenv-echo-env
|
| 890 |
+
coding : Coding Environment (v0.1.0)
|
| 891 |
+
Package: openenv-coding_env
|
| 892 |
+
----------------------------------------------------------------------
|
| 893 |
+
Total: 2 environments
|
| 894 |
+
"""
|
| 895 |
+
discovery = get_discovery()
|
| 896 |
+
discovery.list_environments()
|
src/openenv/cli/__init__.py
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Meta Platforms, Inc. and affiliates.
|
| 2 |
+
# All rights reserved.
|
| 3 |
+
#
|
| 4 |
+
# This source code is licensed under the BSD-style license found in the
|
| 5 |
+
# LICENSE file in the root directory of this source tree.
|
| 6 |
+
|
| 7 |
+
"""OpenEnv CLI package."""
|
| 8 |
+
|
| 9 |
+
__version__ = "0.1.0"
|
src/openenv/cli/__main__.py
ADDED
|
@@ -0,0 +1,62 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Meta Platforms, Inc. and affiliates.
|
| 2 |
+
# All rights reserved.
|
| 3 |
+
#
|
| 4 |
+
# This source code is licensed under the BSD-style license found in the
|
| 5 |
+
# LICENSE file in the root directory of this source tree.
|
| 6 |
+
|
| 7 |
+
"""
|
| 8 |
+
OpenEnv CLI entry point.
|
| 9 |
+
|
| 10 |
+
This module provides the main entry point for the OpenEnv command-line interface,
|
| 11 |
+
following the Hugging Face CLI pattern.
|
| 12 |
+
"""
|
| 13 |
+
|
| 14 |
+
import sys
|
| 15 |
+
|
| 16 |
+
import typer
|
| 17 |
+
|
| 18 |
+
from openenv.cli.commands import build, fork, init, push, serve, validate
|
| 19 |
+
|
| 20 |
+
# Create the main CLI app
|
| 21 |
+
app = typer.Typer(
|
| 22 |
+
name="openenv",
|
| 23 |
+
help="OpenEnv - An e2e framework for creating, deploying and using isolated execution environments for agentic RL training",
|
| 24 |
+
no_args_is_help=True,
|
| 25 |
+
)
|
| 26 |
+
|
| 27 |
+
# Register commands
|
| 28 |
+
app.command(name="init", help="Initialize a new OpenEnv environment")(init.init)
|
| 29 |
+
app.command(name="build", help="Build Docker images for OpenEnv environments")(
|
| 30 |
+
build.build
|
| 31 |
+
)
|
| 32 |
+
app.command(
|
| 33 |
+
name="validate", help="Validate environment structure and deployment readiness"
|
| 34 |
+
)(validate.validate)
|
| 35 |
+
app.command(
|
| 36 |
+
name="push",
|
| 37 |
+
help="Push an OpenEnv environment to Hugging Face Spaces or custom registry",
|
| 38 |
+
)(push.push)
|
| 39 |
+
app.command(name="serve", help="Serve environments locally (TODO: Phase 4)")(
|
| 40 |
+
serve.serve
|
| 41 |
+
)
|
| 42 |
+
app.command(
|
| 43 |
+
name="fork",
|
| 44 |
+
help="Fork (duplicate) a Hugging Face Space to your account",
|
| 45 |
+
)(fork.fork)
|
| 46 |
+
|
| 47 |
+
|
| 48 |
+
# Entry point for setuptools
|
| 49 |
+
def main() -> None:
|
| 50 |
+
"""Main entry point for the CLI."""
|
| 51 |
+
try:
|
| 52 |
+
app()
|
| 53 |
+
except KeyboardInterrupt:
|
| 54 |
+
print("\nOperation cancelled by user.")
|
| 55 |
+
sys.exit(130)
|
| 56 |
+
except Exception as e:
|
| 57 |
+
print(f"Error: {e}", file=sys.stderr)
|
| 58 |
+
sys.exit(1)
|
| 59 |
+
|
| 60 |
+
|
| 61 |
+
if __name__ == "__main__":
|
| 62 |
+
main()
|
src/openenv/cli/_cli_utils.py
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Meta Platforms, Inc. and affiliates.
|
| 2 |
+
# All rights reserved.
|
| 3 |
+
#
|
| 4 |
+
# This source code is licensed under the BSD-style license found in the
|
| 5 |
+
# LICENSE file in the root directory of this source tree.
|
| 6 |
+
|
| 7 |
+
"""CLI utilities for OpenEnv command-line interface."""
|
| 8 |
+
|
| 9 |
+
from pathlib import Path
|
| 10 |
+
from typing import List
|
| 11 |
+
|
| 12 |
+
from rich.console import Console
|
| 13 |
+
|
| 14 |
+
# Create a console instance for CLI output
|
| 15 |
+
console = Console()
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
def validate_env_structure(env_dir: Path, strict: bool = False) -> List[str]:
|
| 19 |
+
"""
|
| 20 |
+
Validate that the directory follows OpenEnv environment structure.
|
| 21 |
+
|
| 22 |
+
Args:
|
| 23 |
+
env_dir: Path to environment directory
|
| 24 |
+
strict: If True, enforce all optional requirements
|
| 25 |
+
|
| 26 |
+
Returns:
|
| 27 |
+
List of validation warnings (empty if all checks pass)
|
| 28 |
+
|
| 29 |
+
Raises:
|
| 30 |
+
FileNotFoundError: If required files are missing
|
| 31 |
+
"""
|
| 32 |
+
warnings = []
|
| 33 |
+
|
| 34 |
+
# Required files
|
| 35 |
+
required_files = [
|
| 36 |
+
"openenv.yaml",
|
| 37 |
+
"__init__.py",
|
| 38 |
+
"client.py",
|
| 39 |
+
"models.py",
|
| 40 |
+
"README.md",
|
| 41 |
+
]
|
| 42 |
+
|
| 43 |
+
for file in required_files:
|
| 44 |
+
if not (env_dir / file).exists():
|
| 45 |
+
raise FileNotFoundError(f"Required file missing: {file}")
|
| 46 |
+
|
| 47 |
+
# Dockerfile: must exist in server/ or at env root
|
| 48 |
+
has_root_dockerfile = (env_dir / "Dockerfile").exists()
|
| 49 |
+
has_server_dockerfile = (env_dir / "server" / "Dockerfile").exists()
|
| 50 |
+
|
| 51 |
+
if not has_root_dockerfile and not has_server_dockerfile:
|
| 52 |
+
raise FileNotFoundError(
|
| 53 |
+
"Required file missing: server/Dockerfile or Dockerfile at env root"
|
| 54 |
+
)
|
| 55 |
+
|
| 56 |
+
# When no root Dockerfile, require the traditional server/ layout
|
| 57 |
+
if not has_root_dockerfile:
|
| 58 |
+
server_dir = env_dir / "server"
|
| 59 |
+
if not server_dir.exists() or not server_dir.is_dir():
|
| 60 |
+
raise FileNotFoundError("Required directory missing: server/")
|
| 61 |
+
|
| 62 |
+
for file in ["server/__init__.py", "server/app.py"]:
|
| 63 |
+
if not (env_dir / file).exists():
|
| 64 |
+
raise FileNotFoundError(f"Required file missing: {file}")
|
| 65 |
+
|
| 66 |
+
# Check for dependency management (pyproject.toml required)
|
| 67 |
+
has_pyproject = (env_dir / "pyproject.toml").exists()
|
| 68 |
+
|
| 69 |
+
if not has_pyproject:
|
| 70 |
+
raise FileNotFoundError(
|
| 71 |
+
"No dependency specification found. 'pyproject.toml' is required."
|
| 72 |
+
)
|
| 73 |
+
|
| 74 |
+
# Warnings for recommended structure
|
| 75 |
+
|
| 76 |
+
if not (env_dir / "outputs").exists():
|
| 77 |
+
warnings.append("Recommended directory missing: outputs/")
|
| 78 |
+
|
| 79 |
+
return warnings
|
src/openenv/cli/_validation.py
ADDED
|
@@ -0,0 +1,162 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Meta Platforms, Inc. and affiliates.
|
| 2 |
+
# All rights reserved.
|
| 3 |
+
#
|
| 4 |
+
# This source code is licensed under the BSD-style license found in the
|
| 5 |
+
# LICENSE file in the root directory of this source tree.
|
| 6 |
+
|
| 7 |
+
"""
|
| 8 |
+
Validation utilities for multi-mode deployment readiness.
|
| 9 |
+
|
| 10 |
+
This module provides functions to check if environments are properly
|
| 11 |
+
configured for multi-mode deployment (Docker, direct Python, notebooks, clusters).
|
| 12 |
+
"""
|
| 13 |
+
|
| 14 |
+
import subprocess
|
| 15 |
+
from pathlib import Path
|
| 16 |
+
|
| 17 |
+
try:
|
| 18 |
+
import tomllib
|
| 19 |
+
except ModuleNotFoundError:
|
| 20 |
+
import tomli as tomllib
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
def validate_multi_mode_deployment(env_path: Path) -> tuple[bool, list[str]]:
|
| 24 |
+
"""
|
| 25 |
+
Validate that an environment is ready for multi-mode deployment.
|
| 26 |
+
|
| 27 |
+
Checks:
|
| 28 |
+
1. pyproject.toml exists
|
| 29 |
+
2. uv.lock exists and is up-to-date
|
| 30 |
+
3. pyproject.toml has [project.scripts] with server entry point
|
| 31 |
+
4. server/app.py has a main() function
|
| 32 |
+
5. Required dependencies are present
|
| 33 |
+
|
| 34 |
+
Returns:
|
| 35 |
+
Tuple of (is_valid, list of issues found)
|
| 36 |
+
"""
|
| 37 |
+
issues = []
|
| 38 |
+
|
| 39 |
+
# Check pyproject.toml exists
|
| 40 |
+
pyproject_path = env_path / "pyproject.toml"
|
| 41 |
+
if not pyproject_path.exists():
|
| 42 |
+
issues.append("Missing pyproject.toml")
|
| 43 |
+
return False, issues
|
| 44 |
+
|
| 45 |
+
# Check uv.lock exists
|
| 46 |
+
lockfile_path = env_path / "uv.lock"
|
| 47 |
+
if not lockfile_path.exists():
|
| 48 |
+
issues.append("Missing uv.lock - run 'uv lock' to generate it")
|
| 49 |
+
else:
|
| 50 |
+
# Check if uv.lock is up-to-date (optional, can be expensive)
|
| 51 |
+
# We can add a check using `uv lock --check` if needed
|
| 52 |
+
try:
|
| 53 |
+
result = subprocess.run(
|
| 54 |
+
["uv", "lock", "--check", "--directory", str(env_path)],
|
| 55 |
+
capture_output=True,
|
| 56 |
+
text=True,
|
| 57 |
+
timeout=5,
|
| 58 |
+
)
|
| 59 |
+
if result.returncode != 0:
|
| 60 |
+
issues.append(
|
| 61 |
+
"uv.lock is out of date with pyproject.toml - run 'uv lock' to update"
|
| 62 |
+
)
|
| 63 |
+
except (subprocess.TimeoutExpired, FileNotFoundError):
|
| 64 |
+
# If uv is not available or times out, skip this check
|
| 65 |
+
pass
|
| 66 |
+
|
| 67 |
+
# Parse pyproject.toml
|
| 68 |
+
try:
|
| 69 |
+
with open(pyproject_path, "rb") as f:
|
| 70 |
+
pyproject = tomllib.load(f)
|
| 71 |
+
except Exception as e:
|
| 72 |
+
issues.append(f"Failed to parse pyproject.toml: {e}")
|
| 73 |
+
return False, issues
|
| 74 |
+
|
| 75 |
+
# Check [project.scripts] section
|
| 76 |
+
scripts = pyproject.get("project", {}).get("scripts", {})
|
| 77 |
+
if "server" not in scripts:
|
| 78 |
+
issues.append("Missing [project.scripts] server entry point")
|
| 79 |
+
|
| 80 |
+
# Check server entry point format
|
| 81 |
+
server_entry = scripts.get("server", "")
|
| 82 |
+
if server_entry and ":main" not in server_entry:
|
| 83 |
+
issues.append(
|
| 84 |
+
f"Server entry point should reference main function, got: {server_entry}"
|
| 85 |
+
)
|
| 86 |
+
|
| 87 |
+
# Check required dependencies
|
| 88 |
+
deps = [dep.lower() for dep in pyproject.get("project", {}).get("dependencies", [])]
|
| 89 |
+
has_openenv = any(
|
| 90 |
+
dep.startswith("openenv") and not dep.startswith("openenv-core") for dep in deps
|
| 91 |
+
)
|
| 92 |
+
has_legacy_core = any(dep.startswith("openenv-core") for dep in deps)
|
| 93 |
+
|
| 94 |
+
if not (has_openenv or has_legacy_core):
|
| 95 |
+
issues.append(
|
| 96 |
+
"Missing required dependency: openenv-core>=0.2.0 (or openenv>=0.2.0)"
|
| 97 |
+
)
|
| 98 |
+
|
| 99 |
+
# Check server/app.py exists
|
| 100 |
+
server_app = env_path / "server" / "app.py"
|
| 101 |
+
if not server_app.exists():
|
| 102 |
+
issues.append("Missing server/app.py")
|
| 103 |
+
else:
|
| 104 |
+
# Check for main() function (flexible - with or without parameters)
|
| 105 |
+
app_content = server_app.read_text(encoding="utf-8")
|
| 106 |
+
if "def main(" not in app_content:
|
| 107 |
+
issues.append("server/app.py missing main() function")
|
| 108 |
+
|
| 109 |
+
# Check if main() is callable
|
| 110 |
+
if "__name__" not in app_content or "main()" not in app_content:
|
| 111 |
+
issues.append(
|
| 112 |
+
"server/app.py main() function not callable (missing if __name__ == '__main__')"
|
| 113 |
+
)
|
| 114 |
+
|
| 115 |
+
return len(issues) == 0, issues
|
| 116 |
+
|
| 117 |
+
|
| 118 |
+
def get_deployment_modes(env_path: Path) -> dict[str, bool]:
|
| 119 |
+
"""
|
| 120 |
+
Check which deployment modes are supported by the environment.
|
| 121 |
+
|
| 122 |
+
Returns:
|
| 123 |
+
Dictionary with deployment mode names and whether they're supported
|
| 124 |
+
"""
|
| 125 |
+
modes = {
|
| 126 |
+
"docker": False,
|
| 127 |
+
"openenv_serve": False,
|
| 128 |
+
"uv_run": False,
|
| 129 |
+
"python_module": False,
|
| 130 |
+
}
|
| 131 |
+
|
| 132 |
+
# Check Docker (Dockerfile may be in server/ or at env root)
|
| 133 |
+
modes["docker"] = (env_path / "server" / "Dockerfile").exists() or (
|
| 134 |
+
env_path / "Dockerfile"
|
| 135 |
+
).exists()
|
| 136 |
+
|
| 137 |
+
# Check multi-mode deployment readiness
|
| 138 |
+
is_valid, _ = validate_multi_mode_deployment(env_path)
|
| 139 |
+
if is_valid:
|
| 140 |
+
modes["openenv_serve"] = True
|
| 141 |
+
modes["uv_run"] = True
|
| 142 |
+
modes["python_module"] = True
|
| 143 |
+
|
| 144 |
+
return modes
|
| 145 |
+
|
| 146 |
+
|
| 147 |
+
def format_validation_report(env_name: str, is_valid: bool, issues: list[str]) -> str:
|
| 148 |
+
"""
|
| 149 |
+
Format a validation report for display.
|
| 150 |
+
|
| 151 |
+
Returns:
|
| 152 |
+
Formatted report string
|
| 153 |
+
"""
|
| 154 |
+
if is_valid:
|
| 155 |
+
return f"[OK] {env_name}: Ready for multi-mode deployment"
|
| 156 |
+
|
| 157 |
+
report = [f"[FAIL] {env_name}: Not ready for multi-mode deployment", ""]
|
| 158 |
+
report.append("Issues found:")
|
| 159 |
+
for issue in issues:
|
| 160 |
+
report.append(f" - {issue}")
|
| 161 |
+
|
| 162 |
+
return "\n".join(report)
|
src/openenv/cli/commands/__init__.py
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Meta Platforms, Inc. and affiliates.
|
| 2 |
+
# All rights reserved.
|
| 3 |
+
#
|
| 4 |
+
# This source code is licensed under the BSD-style license found in the
|
| 5 |
+
# LICENSE file in the root directory of this source tree.
|
| 6 |
+
|
| 7 |
+
"""OpenEnv CLI commands."""
|
| 8 |
+
|
| 9 |
+
from . import build, fork, init, push, serve, validate
|
| 10 |
+
|
| 11 |
+
__all__ = ["build", "fork", "init", "push", "serve", "validate"]
|
src/openenv/cli/commands/build.py
ADDED
|
@@ -0,0 +1,461 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Meta Platforms, Inc. and affiliates.
|
| 2 |
+
# All rights reserved.
|
| 3 |
+
#
|
| 4 |
+
# This source code is licensed under the BSD-style license found in the
|
| 5 |
+
# LICENSE file in the root directory of this source tree.
|
| 6 |
+
|
| 7 |
+
"""Build Docker images for OpenEnv environments."""
|
| 8 |
+
|
| 9 |
+
from __future__ import annotations
|
| 10 |
+
|
| 11 |
+
import shutil
|
| 12 |
+
import subprocess
|
| 13 |
+
import tempfile
|
| 14 |
+
import sys
|
| 15 |
+
from pathlib import Path
|
| 16 |
+
from typing import Annotated
|
| 17 |
+
|
| 18 |
+
import typer
|
| 19 |
+
|
| 20 |
+
from .._cli_utils import console
|
| 21 |
+
|
| 22 |
+
app = typer.Typer(help="Build Docker images for OpenEnv environments")
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
def _detect_build_context(env_path: Path) -> tuple[str, Path, Path | None]:
|
| 26 |
+
"""
|
| 27 |
+
Detect whether we're building a standalone or in-repo environment.
|
| 28 |
+
|
| 29 |
+
Returns:
|
| 30 |
+
tuple: (build_mode, build_context_path, repo_root)
|
| 31 |
+
- build_mode: "standalone" or "in-repo"
|
| 32 |
+
- build_context_path: Path to use as Docker build context
|
| 33 |
+
- repo_root: Path to repo root (None for standalone)
|
| 34 |
+
"""
|
| 35 |
+
# Ensure env_path is absolute for proper comparison
|
| 36 |
+
env_path = env_path.absolute()
|
| 37 |
+
|
| 38 |
+
# Check if we're in a git repository
|
| 39 |
+
current = env_path
|
| 40 |
+
repo_root = None
|
| 41 |
+
|
| 42 |
+
# Walk up to find .git directory
|
| 43 |
+
for parent in [current] + list(current.parents):
|
| 44 |
+
if (parent / ".git").exists():
|
| 45 |
+
repo_root = parent
|
| 46 |
+
break
|
| 47 |
+
|
| 48 |
+
if repo_root is None:
|
| 49 |
+
# Not in a git repo = standalone
|
| 50 |
+
return "standalone", env_path, None
|
| 51 |
+
|
| 52 |
+
# Check if environment is under envs/ (in-repo pattern)
|
| 53 |
+
try:
|
| 54 |
+
rel_path = env_path.relative_to(repo_root)
|
| 55 |
+
rel_str = str(rel_path)
|
| 56 |
+
if (
|
| 57 |
+
rel_str.startswith("envs/")
|
| 58 |
+
or rel_str.startswith("envs\\")
|
| 59 |
+
or rel_str.startswith("envs/")
|
| 60 |
+
):
|
| 61 |
+
# In-repo environment
|
| 62 |
+
return "in-repo", repo_root, repo_root
|
| 63 |
+
except ValueError:
|
| 64 |
+
pass
|
| 65 |
+
|
| 66 |
+
# Otherwise, it's standalone (environment outside repo structure)
|
| 67 |
+
return "standalone", env_path, None
|
| 68 |
+
|
| 69 |
+
|
| 70 |
+
def _prepare_standalone_build(env_path: Path, temp_dir: Path) -> Path:
|
| 71 |
+
"""
|
| 72 |
+
Prepare a standalone environment for building.
|
| 73 |
+
|
| 74 |
+
For standalone builds:
|
| 75 |
+
1. Copy environment to temp directory
|
| 76 |
+
2. Ensure pyproject.toml depends on openenv
|
| 77 |
+
|
| 78 |
+
Returns:
|
| 79 |
+
Path to the prepared build directory
|
| 80 |
+
"""
|
| 81 |
+
console.print("[cyan]Preparing standalone build...[/cyan]")
|
| 82 |
+
|
| 83 |
+
# Copy environment to temp directory
|
| 84 |
+
build_dir = temp_dir / env_path.name
|
| 85 |
+
shutil.copytree(env_path, build_dir, symlinks=True)
|
| 86 |
+
|
| 87 |
+
console.print(f"[cyan]Copied environment to:[/cyan] {build_dir}")
|
| 88 |
+
|
| 89 |
+
# Check if pyproject.toml has openenv dependency
|
| 90 |
+
pyproject_path = build_dir / "pyproject.toml"
|
| 91 |
+
if pyproject_path.exists():
|
| 92 |
+
with open(pyproject_path, "rb") as f:
|
| 93 |
+
try:
|
| 94 |
+
import tomli
|
| 95 |
+
|
| 96 |
+
pyproject = tomli.load(f)
|
| 97 |
+
deps = pyproject.get("project", {}).get("dependencies", [])
|
| 98 |
+
|
| 99 |
+
# Check if openenv dependency is declared
|
| 100 |
+
has_openenv = any(dep.startswith("openenv") for dep in deps)
|
| 101 |
+
|
| 102 |
+
if not has_openenv:
|
| 103 |
+
console.print(
|
| 104 |
+
"[yellow]Warning:[/yellow] pyproject.toml doesn't list the openenv dependency",
|
| 105 |
+
)
|
| 106 |
+
console.print(
|
| 107 |
+
"[yellow]You may need to add:[/yellow] openenv>=0.2.0",
|
| 108 |
+
)
|
| 109 |
+
except ImportError:
|
| 110 |
+
console.print(
|
| 111 |
+
"[yellow]Warning:[/yellow] tomli not available, skipping dependency check",
|
| 112 |
+
)
|
| 113 |
+
|
| 114 |
+
return build_dir
|
| 115 |
+
|
| 116 |
+
|
| 117 |
+
def _prepare_inrepo_build(env_path: Path, repo_root: Path, temp_dir: Path) -> Path:
|
| 118 |
+
"""
|
| 119 |
+
Prepare an in-repo environment for building.
|
| 120 |
+
|
| 121 |
+
For in-repo builds:
|
| 122 |
+
1. Create temp directory with environment and core
|
| 123 |
+
2. Set up structure that matches expected layout
|
| 124 |
+
|
| 125 |
+
Returns:
|
| 126 |
+
Path to the prepared build directory
|
| 127 |
+
"""
|
| 128 |
+
console.print("[cyan]Preparing in-repo build...[/cyan]")
|
| 129 |
+
|
| 130 |
+
# Copy environment to temp directory
|
| 131 |
+
build_dir = temp_dir / env_path.name
|
| 132 |
+
shutil.copytree(env_path, build_dir, symlinks=True)
|
| 133 |
+
|
| 134 |
+
# Copy OpenEnv package metadata + sources to temp directory.
|
| 135 |
+
# Keep the src/ layout since pyproject.toml uses package-dir = {"" = "src"}.
|
| 136 |
+
package_src = repo_root / "src" / "openenv"
|
| 137 |
+
package_dest = build_dir / "openenv"
|
| 138 |
+
if package_src.exists():
|
| 139 |
+
package_dest.mkdir(parents=True, exist_ok=True)
|
| 140 |
+
shutil.copytree(package_src, package_dest / "src" / "openenv", symlinks=True)
|
| 141 |
+
|
| 142 |
+
for filename in ("pyproject.toml", "README.md"):
|
| 143 |
+
src_file = repo_root / filename
|
| 144 |
+
if src_file.exists():
|
| 145 |
+
shutil.copy2(src_file, package_dest / filename)
|
| 146 |
+
|
| 147 |
+
console.print(f"[cyan]Copied OpenEnv package to:[/cyan] {package_dest}")
|
| 148 |
+
|
| 149 |
+
# Update pyproject.toml to reference local OpenEnv copy
|
| 150 |
+
pyproject_path = build_dir / "pyproject.toml"
|
| 151 |
+
if pyproject_path.exists():
|
| 152 |
+
with open(pyproject_path, "rb") as f:
|
| 153 |
+
try:
|
| 154 |
+
import tomli
|
| 155 |
+
|
| 156 |
+
pyproject = tomli.load(f)
|
| 157 |
+
deps = pyproject.get("project", {}).get("dependencies", [])
|
| 158 |
+
|
| 159 |
+
# Replace openenv/openenv-core with local reference
|
| 160 |
+
new_deps = []
|
| 161 |
+
for dep in deps:
|
| 162 |
+
if (
|
| 163 |
+
dep.startswith("openenv-core")
|
| 164 |
+
or dep.startswith("openenv_core")
|
| 165 |
+
or dep.startswith("openenv")
|
| 166 |
+
):
|
| 167 |
+
# Skip - we'll use local core
|
| 168 |
+
continue
|
| 169 |
+
new_deps.append(dep)
|
| 170 |
+
|
| 171 |
+
# Write back with local core reference
|
| 172 |
+
pyproject["project"]["dependencies"] = new_deps + [
|
| 173 |
+
"openenv-core @ file:///app/env/openenv"
|
| 174 |
+
]
|
| 175 |
+
|
| 176 |
+
# Write updated pyproject.toml
|
| 177 |
+
with open(pyproject_path, "wb") as out_f:
|
| 178 |
+
import tomli_w
|
| 179 |
+
|
| 180 |
+
tomli_w.dump(pyproject, out_f)
|
| 181 |
+
|
| 182 |
+
console.print(
|
| 183 |
+
"[cyan]Updated pyproject.toml to use local core[/cyan]"
|
| 184 |
+
)
|
| 185 |
+
|
| 186 |
+
# Remove old lockfile since dependencies changed
|
| 187 |
+
lockfile = build_dir / "uv.lock"
|
| 188 |
+
if lockfile.exists():
|
| 189 |
+
lockfile.unlink()
|
| 190 |
+
console.print("[cyan]Removed outdated uv.lock[/cyan]")
|
| 191 |
+
|
| 192 |
+
except ImportError:
|
| 193 |
+
console.print(
|
| 194 |
+
"[yellow]Warning:[/yellow] tomli/tomli_w not available, using pyproject.toml as-is",
|
| 195 |
+
)
|
| 196 |
+
else:
|
| 197 |
+
console.print(
|
| 198 |
+
"[yellow]Warning:[/yellow] OpenEnv package not found, building without it"
|
| 199 |
+
)
|
| 200 |
+
|
| 201 |
+
console.print(f"[cyan]Build directory prepared:[/cyan] {build_dir}")
|
| 202 |
+
return build_dir
|
| 203 |
+
|
| 204 |
+
|
| 205 |
+
def _run_command(
|
| 206 |
+
cmd: list[str],
|
| 207 |
+
cwd: Path | None = None,
|
| 208 |
+
check: bool = True,
|
| 209 |
+
) -> subprocess.CompletedProcess:
|
| 210 |
+
"""Run a shell command and handle errors."""
|
| 211 |
+
console.print(f"[bold cyan]Running:[/bold cyan] {' '.join(cmd)}")
|
| 212 |
+
try:
|
| 213 |
+
result = subprocess.run(
|
| 214 |
+
cmd, cwd=cwd, check=check, capture_output=True, text=True
|
| 215 |
+
)
|
| 216 |
+
if result.stdout:
|
| 217 |
+
console.print(result.stdout)
|
| 218 |
+
if result.stderr:
|
| 219 |
+
print(result.stderr, file=sys.stderr)
|
| 220 |
+
return result
|
| 221 |
+
except subprocess.CalledProcessError as e:
|
| 222 |
+
print(f"Error running command: {e}", file=sys.stderr)
|
| 223 |
+
if e.stdout:
|
| 224 |
+
console.print(e.stdout)
|
| 225 |
+
if e.stderr:
|
| 226 |
+
print(e.stderr, file=sys.stderr)
|
| 227 |
+
if check:
|
| 228 |
+
raise typer.Exit(1) from e
|
| 229 |
+
return e
|
| 230 |
+
|
| 231 |
+
|
| 232 |
+
def _build_docker_image(
|
| 233 |
+
env_path: Path,
|
| 234 |
+
tag: str | None = None,
|
| 235 |
+
context_path: Path | None = None,
|
| 236 |
+
dockerfile: Path | None = None,
|
| 237 |
+
build_args: dict[str, str] | None = None,
|
| 238 |
+
no_cache: bool = False,
|
| 239 |
+
) -> bool:
|
| 240 |
+
"""Build Docker image for the environment with smart context detection."""
|
| 241 |
+
|
| 242 |
+
# Detect build context (standalone vs in-repo)
|
| 243 |
+
build_mode, detected_context, repo_root = _detect_build_context(env_path)
|
| 244 |
+
|
| 245 |
+
console.print(f"[bold cyan]Build mode detected:[/bold cyan] {build_mode}")
|
| 246 |
+
|
| 247 |
+
# Use detected context unless explicitly overridden
|
| 248 |
+
if context_path is None:
|
| 249 |
+
context_path = detected_context
|
| 250 |
+
|
| 251 |
+
# Create temporary build directory
|
| 252 |
+
with tempfile.TemporaryDirectory() as temp_dir_str:
|
| 253 |
+
temp_dir = Path(temp_dir_str)
|
| 254 |
+
|
| 255 |
+
# Prepare build directory based on mode
|
| 256 |
+
if build_mode == "standalone":
|
| 257 |
+
build_dir = _prepare_standalone_build(env_path, temp_dir)
|
| 258 |
+
else: # in-repo
|
| 259 |
+
build_dir = _prepare_inrepo_build(env_path, repo_root, temp_dir)
|
| 260 |
+
|
| 261 |
+
# Determine Dockerfile path
|
| 262 |
+
if dockerfile is None:
|
| 263 |
+
# Look for Dockerfile in server/ subdirectory
|
| 264 |
+
dockerfile = build_dir / "server" / "Dockerfile"
|
| 265 |
+
if not dockerfile.exists():
|
| 266 |
+
# Fallback to root of build directory
|
| 267 |
+
dockerfile = build_dir / "Dockerfile"
|
| 268 |
+
|
| 269 |
+
if not dockerfile.exists():
|
| 270 |
+
console.print(
|
| 271 |
+
f"[bold red]Error:[/bold red] Dockerfile not found at {dockerfile}",
|
| 272 |
+
)
|
| 273 |
+
return False
|
| 274 |
+
|
| 275 |
+
# Generate tag if not provided
|
| 276 |
+
if tag is None:
|
| 277 |
+
env_name = env_path.name
|
| 278 |
+
if env_name.endswith("_env"):
|
| 279 |
+
env_name = env_name[:-4]
|
| 280 |
+
tag = f"openenv-{env_name}"
|
| 281 |
+
|
| 282 |
+
console.print(f"[bold cyan]Building Docker image:[/bold cyan] {tag}")
|
| 283 |
+
console.print(f"[bold cyan]Build context:[/bold cyan] {build_dir}")
|
| 284 |
+
console.print(f"[bold cyan]Dockerfile:[/bold cyan] {dockerfile}")
|
| 285 |
+
|
| 286 |
+
# Prepare build args
|
| 287 |
+
if build_args is None:
|
| 288 |
+
build_args = {}
|
| 289 |
+
|
| 290 |
+
# Add build mode and env name to build args
|
| 291 |
+
build_args["BUILD_MODE"] = build_mode
|
| 292 |
+
build_args["ENV_NAME"] = env_path.name.replace("_env", "")
|
| 293 |
+
|
| 294 |
+
# Build Docker command
|
| 295 |
+
cmd = ["docker", "build", "-t", tag, "-f", str(dockerfile)]
|
| 296 |
+
|
| 297 |
+
if no_cache:
|
| 298 |
+
cmd.append("--no-cache")
|
| 299 |
+
|
| 300 |
+
for key, value in build_args.items():
|
| 301 |
+
cmd.extend(["--build-arg", f"{key}={value}"])
|
| 302 |
+
|
| 303 |
+
cmd.append(str(build_dir))
|
| 304 |
+
|
| 305 |
+
result = _run_command(cmd, check=False)
|
| 306 |
+
return result.returncode == 0
|
| 307 |
+
|
| 308 |
+
|
| 309 |
+
def _push_docker_image(tag: str, registry: str | None = None) -> bool:
|
| 310 |
+
"""Push Docker image to registry."""
|
| 311 |
+
if registry:
|
| 312 |
+
full_tag = f"{registry}/{tag}"
|
| 313 |
+
console.print(f"[bold cyan]Tagging image as {full_tag}[/bold cyan]")
|
| 314 |
+
_run_command(["docker", "tag", tag, full_tag])
|
| 315 |
+
tag = full_tag
|
| 316 |
+
|
| 317 |
+
console.print(f"[bold cyan]Pushing image:[/bold cyan] {tag}")
|
| 318 |
+
result = _run_command(["docker", "push", tag], check=False)
|
| 319 |
+
return result.returncode == 0
|
| 320 |
+
|
| 321 |
+
|
| 322 |
+
@app.command()
|
| 323 |
+
def build(
|
| 324 |
+
env_path: Annotated[
|
| 325 |
+
str | None,
|
| 326 |
+
typer.Argument(
|
| 327 |
+
help="Path to the environment directory (default: current directory)"
|
| 328 |
+
),
|
| 329 |
+
] = None,
|
| 330 |
+
tag: Annotated[
|
| 331 |
+
str | None,
|
| 332 |
+
typer.Option(
|
| 333 |
+
"--tag",
|
| 334 |
+
"-t",
|
| 335 |
+
help="Docker image tag (default: openenv-<env_name>)",
|
| 336 |
+
),
|
| 337 |
+
] = None,
|
| 338 |
+
context: Annotated[
|
| 339 |
+
str | None,
|
| 340 |
+
typer.Option(
|
| 341 |
+
"--context",
|
| 342 |
+
"-c",
|
| 343 |
+
help="Build context path (default: <env_path>/server)",
|
| 344 |
+
),
|
| 345 |
+
] = None,
|
| 346 |
+
dockerfile: Annotated[
|
| 347 |
+
str | None,
|
| 348 |
+
typer.Option(
|
| 349 |
+
"--dockerfile",
|
| 350 |
+
"-f",
|
| 351 |
+
help="Path to Dockerfile (default: <context>/Dockerfile)",
|
| 352 |
+
),
|
| 353 |
+
] = None,
|
| 354 |
+
no_cache: Annotated[
|
| 355 |
+
bool,
|
| 356 |
+
typer.Option(
|
| 357 |
+
"--no-cache",
|
| 358 |
+
help="Build without using cache",
|
| 359 |
+
),
|
| 360 |
+
] = False,
|
| 361 |
+
build_arg: Annotated[
|
| 362 |
+
list[str] | None,
|
| 363 |
+
typer.Option(
|
| 364 |
+
"--build-arg",
|
| 365 |
+
help="Build arguments (can be used multiple times, format: KEY=VALUE)",
|
| 366 |
+
),
|
| 367 |
+
] = None,
|
| 368 |
+
) -> None:
|
| 369 |
+
"""
|
| 370 |
+
Build Docker images for OpenEnv environments.
|
| 371 |
+
|
| 372 |
+
This command builds Docker images using the environment's pyproject.toml
|
| 373 |
+
and uv for dependency management. Run from the environment root directory.
|
| 374 |
+
|
| 375 |
+
Examples:
|
| 376 |
+
# Build from environment root (recommended)
|
| 377 |
+
$ cd my_env
|
| 378 |
+
$ openenv build
|
| 379 |
+
|
| 380 |
+
# Build with custom tag
|
| 381 |
+
$ openenv build -t my-custom-tag
|
| 382 |
+
|
| 383 |
+
# Build without cache
|
| 384 |
+
$ openenv build --no-cache
|
| 385 |
+
|
| 386 |
+
# Build with custom build arguments
|
| 387 |
+
$ openenv build --build-arg VERSION=1.0 --build-arg ENV=prod
|
| 388 |
+
|
| 389 |
+
# Build from different directory
|
| 390 |
+
$ openenv build envs/echo_env
|
| 391 |
+
"""
|
| 392 |
+
# Determine environment path (default to current directory)
|
| 393 |
+
if env_path is None:
|
| 394 |
+
env_path_obj = Path.cwd()
|
| 395 |
+
else:
|
| 396 |
+
env_path_obj = Path(env_path)
|
| 397 |
+
|
| 398 |
+
# Validate environment path
|
| 399 |
+
if not env_path_obj.exists():
|
| 400 |
+
print(
|
| 401 |
+
f"Error: Environment path does not exist: {env_path_obj}",
|
| 402 |
+
file=sys.stderr,
|
| 403 |
+
)
|
| 404 |
+
raise typer.Exit(1)
|
| 405 |
+
|
| 406 |
+
if not env_path_obj.is_dir():
|
| 407 |
+
print(
|
| 408 |
+
f"Error: Environment path is not a directory: {env_path_obj}",
|
| 409 |
+
file=sys.stderr,
|
| 410 |
+
)
|
| 411 |
+
raise typer.Exit(1)
|
| 412 |
+
|
| 413 |
+
# Check for openenv.yaml to confirm this is an environment directory
|
| 414 |
+
openenv_yaml = env_path_obj / "openenv.yaml"
|
| 415 |
+
if not openenv_yaml.exists():
|
| 416 |
+
print(
|
| 417 |
+
f"Error: Not an OpenEnv environment directory (missing openenv.yaml): {env_path_obj}",
|
| 418 |
+
file=sys.stderr,
|
| 419 |
+
)
|
| 420 |
+
print(
|
| 421 |
+
"Hint: Run this command from the environment root directory or specify the path",
|
| 422 |
+
file=sys.stderr,
|
| 423 |
+
)
|
| 424 |
+
raise typer.Exit(1)
|
| 425 |
+
|
| 426 |
+
console.print(f"[bold]Building Docker image for:[/bold] {env_path_obj.name}")
|
| 427 |
+
console.print("=" * 60)
|
| 428 |
+
|
| 429 |
+
# Parse build args
|
| 430 |
+
build_args = {}
|
| 431 |
+
if build_arg:
|
| 432 |
+
for arg in build_arg:
|
| 433 |
+
if "=" in arg:
|
| 434 |
+
key, value = arg.split("=", 1)
|
| 435 |
+
build_args[key] = value
|
| 436 |
+
else:
|
| 437 |
+
print(
|
| 438 |
+
f"Warning: Invalid build arg format: {arg}",
|
| 439 |
+
file=sys.stderr,
|
| 440 |
+
)
|
| 441 |
+
|
| 442 |
+
# Convert string paths to Path objects
|
| 443 |
+
context_path_obj = Path(context) if context else None
|
| 444 |
+
dockerfile_path_obj = Path(dockerfile) if dockerfile else None
|
| 445 |
+
|
| 446 |
+
# Build Docker image
|
| 447 |
+
success = _build_docker_image(
|
| 448 |
+
env_path=env_path_obj,
|
| 449 |
+
tag=tag,
|
| 450 |
+
context_path=context_path_obj,
|
| 451 |
+
dockerfile=dockerfile_path_obj,
|
| 452 |
+
build_args=build_args if build_args else None,
|
| 453 |
+
no_cache=no_cache,
|
| 454 |
+
)
|
| 455 |
+
|
| 456 |
+
if not success:
|
| 457 |
+
print("✗ Docker build failed", file=sys.stderr)
|
| 458 |
+
raise typer.Exit(1)
|
| 459 |
+
|
| 460 |
+
console.print("[bold green]✓ Docker build successful[/bold green]")
|
| 461 |
+
console.print("\n[bold green]Done![/bold green]")
|
src/openenv/cli/commands/fork.py
ADDED
|
@@ -0,0 +1,197 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Meta Platforms, Inc. and affiliates.
|
| 2 |
+
# All rights reserved.
|
| 3 |
+
#
|
| 4 |
+
# This source code is licensed under the BSD-style license found in the
|
| 5 |
+
# LICENSE file in the root directory of this source tree.
|
| 6 |
+
|
| 7 |
+
"""Fork (duplicate) a Hugging Face Space using the Hub API."""
|
| 8 |
+
|
| 9 |
+
from __future__ import annotations
|
| 10 |
+
|
| 11 |
+
from typing import Annotated
|
| 12 |
+
|
| 13 |
+
import typer
|
| 14 |
+
from huggingface_hub import HfApi, login, whoami
|
| 15 |
+
|
| 16 |
+
from .._cli_utils import console
|
| 17 |
+
|
| 18 |
+
app = typer.Typer(
|
| 19 |
+
help="Fork (duplicate) an OpenEnv environment on Hugging Face to your account"
|
| 20 |
+
)
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
def _parse_key_value(s: str) -> tuple[str, str]:
|
| 24 |
+
"""Parse KEY=VALUE string. Raises BadParameter if no '='."""
|
| 25 |
+
if "=" not in s:
|
| 26 |
+
raise typer.BadParameter(
|
| 27 |
+
f"Expected KEY=VALUE format, got: {s!r}. "
|
| 28 |
+
"Use --set-env KEY=VALUE or --set-secret KEY=VALUE"
|
| 29 |
+
)
|
| 30 |
+
key, _, value = s.partition("=")
|
| 31 |
+
key = key.strip()
|
| 32 |
+
if not key:
|
| 33 |
+
raise typer.BadParameter(f"Empty key in: {s!r}")
|
| 34 |
+
return key, value.strip()
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
def _ensure_hf_authenticated() -> str:
|
| 38 |
+
"""Ensure user is authenticated with Hugging Face. Returns username."""
|
| 39 |
+
try:
|
| 40 |
+
user_info = whoami()
|
| 41 |
+
if isinstance(user_info, dict):
|
| 42 |
+
username = (
|
| 43 |
+
user_info.get("name")
|
| 44 |
+
or user_info.get("fullname")
|
| 45 |
+
or user_info.get("username")
|
| 46 |
+
)
|
| 47 |
+
else:
|
| 48 |
+
username = (
|
| 49 |
+
getattr(user_info, "name", None)
|
| 50 |
+
or getattr(user_info, "fullname", None)
|
| 51 |
+
or getattr(user_info, "username", None)
|
| 52 |
+
)
|
| 53 |
+
if not username:
|
| 54 |
+
raise ValueError("Could not extract username from whoami response")
|
| 55 |
+
console.print(f"[bold green]✓[/bold green] Authenticated as: {username}")
|
| 56 |
+
return username
|
| 57 |
+
except Exception:
|
| 58 |
+
console.print(
|
| 59 |
+
"[bold yellow]Not authenticated with Hugging Face. Please login...[/bold yellow]"
|
| 60 |
+
)
|
| 61 |
+
try:
|
| 62 |
+
login()
|
| 63 |
+
user_info = whoami()
|
| 64 |
+
if isinstance(user_info, dict):
|
| 65 |
+
username = (
|
| 66 |
+
user_info.get("name")
|
| 67 |
+
or user_info.get("fullname")
|
| 68 |
+
or user_info.get("username")
|
| 69 |
+
)
|
| 70 |
+
else:
|
| 71 |
+
username = (
|
| 72 |
+
getattr(user_info, "name", None)
|
| 73 |
+
or getattr(user_info, "fullname", None)
|
| 74 |
+
or getattr(user_info, "username", None)
|
| 75 |
+
)
|
| 76 |
+
if not username:
|
| 77 |
+
raise ValueError("Could not extract username from whoami response")
|
| 78 |
+
console.print(f"[bold green]✓[/bold green] Authenticated as: {username}")
|
| 79 |
+
return username
|
| 80 |
+
except Exception as e:
|
| 81 |
+
raise typer.BadParameter(
|
| 82 |
+
f"Hugging Face authentication failed: {e}. Please run login manually."
|
| 83 |
+
) from e
|
| 84 |
+
|
| 85 |
+
|
| 86 |
+
@app.command()
|
| 87 |
+
def fork(
|
| 88 |
+
source_space: Annotated[
|
| 89 |
+
str,
|
| 90 |
+
typer.Argument(
|
| 91 |
+
help="Source Space ID in format 'owner/space-name' (e.g. org/my-openenv-space)"
|
| 92 |
+
),
|
| 93 |
+
],
|
| 94 |
+
repo_id: Annotated[
|
| 95 |
+
str | None,
|
| 96 |
+
typer.Option(
|
| 97 |
+
"--repo-id",
|
| 98 |
+
"-r",
|
| 99 |
+
help="Target repo ID for the fork (default: created under your account with same name)",
|
| 100 |
+
),
|
| 101 |
+
] = None,
|
| 102 |
+
private: Annotated[
|
| 103 |
+
bool,
|
| 104 |
+
typer.Option("--private", help="Create the forked Space as private"),
|
| 105 |
+
] = False,
|
| 106 |
+
set_env: Annotated[
|
| 107 |
+
list[str],
|
| 108 |
+
typer.Option(
|
| 109 |
+
"--set-env",
|
| 110 |
+
"-e",
|
| 111 |
+
help="Set Space variable (public). Can be repeated. Format: KEY=VALUE",
|
| 112 |
+
),
|
| 113 |
+
] = [],
|
| 114 |
+
set_secret: Annotated[
|
| 115 |
+
list[str],
|
| 116 |
+
typer.Option(
|
| 117 |
+
"--set-secret",
|
| 118 |
+
"--secret",
|
| 119 |
+
"-s",
|
| 120 |
+
help="Set Space secret. Can be repeated. Format: KEY=VALUE",
|
| 121 |
+
),
|
| 122 |
+
] = [],
|
| 123 |
+
hardware: Annotated[
|
| 124 |
+
str | None,
|
| 125 |
+
typer.Option(
|
| 126 |
+
"--hardware",
|
| 127 |
+
"-H",
|
| 128 |
+
help="Request hardware (e.g. t4-medium, cpu-basic). See Hub docs for options.",
|
| 129 |
+
),
|
| 130 |
+
] = None,
|
| 131 |
+
) -> None:
|
| 132 |
+
"""
|
| 133 |
+
Fork (duplicate) a Hugging Face Space to your account using the Hub API.
|
| 134 |
+
|
| 135 |
+
Uses the Hugging Face duplicate_space API. You can set environment variables
|
| 136 |
+
and secrets, and request hardware/storage/sleep time at creation time.
|
| 137 |
+
|
| 138 |
+
Examples:
|
| 139 |
+
$ openenv fork owner/source-space
|
| 140 |
+
$ openenv fork owner/source-space --private
|
| 141 |
+
$ openenv fork owner/source-space --repo-id myuser/my-fork
|
| 142 |
+
$ openenv fork owner/source-space --set-env MODEL_ID=user/model --set-secret HF_TOKEN=hf_xxx
|
| 143 |
+
$ openenv fork owner/source-space --hardware t4-medium
|
| 144 |
+
"""
|
| 145 |
+
if "/" not in source_space or source_space.count("/") != 1:
|
| 146 |
+
raise typer.BadParameter(
|
| 147 |
+
f"Invalid source Space ID: {source_space!r}. Expected format: 'owner/space-name'"
|
| 148 |
+
)
|
| 149 |
+
|
| 150 |
+
_ensure_hf_authenticated()
|
| 151 |
+
api = HfApi()
|
| 152 |
+
|
| 153 |
+
# Build kwargs for duplicate_space (only pass what we have)
|
| 154 |
+
dup_kwargs: dict = {
|
| 155 |
+
"from_id": source_space,
|
| 156 |
+
"private": private,
|
| 157 |
+
}
|
| 158 |
+
if set_env:
|
| 159 |
+
dup_kwargs["variables"] = [
|
| 160 |
+
{"key": k, "value": v} for k, v in (_parse_key_value(x) for x in set_env)
|
| 161 |
+
]
|
| 162 |
+
if set_secret:
|
| 163 |
+
dup_kwargs["secrets"] = [
|
| 164 |
+
{"key": k, "value": v} for k, v in (_parse_key_value(x) for x in set_secret)
|
| 165 |
+
]
|
| 166 |
+
# HF API requires hardware when duplicating; default to free cpu-basic
|
| 167 |
+
dup_kwargs["hardware"] = hardware if hardware is not None else "cpu-basic"
|
| 168 |
+
if repo_id is not None:
|
| 169 |
+
if "/" not in repo_id or repo_id.count("/") != 1:
|
| 170 |
+
raise typer.BadParameter(
|
| 171 |
+
f"Invalid --repo-id: {repo_id!r}. Expected format: 'username/repo-name'"
|
| 172 |
+
)
|
| 173 |
+
dup_kwargs["to_id"] = repo_id
|
| 174 |
+
|
| 175 |
+
console.print(f"[bold cyan]Forking Space {source_space}...[/bold cyan]")
|
| 176 |
+
try:
|
| 177 |
+
result = api.duplicate_space(**dup_kwargs)
|
| 178 |
+
except Exception as e:
|
| 179 |
+
console.print(f"[bold red]✗[/bold red] Fork failed: {e}")
|
| 180 |
+
raise typer.Exit(1) from e
|
| 181 |
+
|
| 182 |
+
# result is RepoUrl (str-like) or similar; get repo_id for display
|
| 183 |
+
if hasattr(result, "repo_id"):
|
| 184 |
+
new_repo_id = result.repo_id
|
| 185 |
+
elif isinstance(result, str):
|
| 186 |
+
# URL like https://huggingface.co/spaces/owner/name -> owner/name
|
| 187 |
+
if "/spaces/" in result:
|
| 188 |
+
new_repo_id = result.split("/spaces/")[-1].rstrip("/")
|
| 189 |
+
else:
|
| 190 |
+
new_repo_id = result
|
| 191 |
+
else:
|
| 192 |
+
new_repo_id = getattr(result, "repo_id", str(result))
|
| 193 |
+
|
| 194 |
+
console.print("[bold green]✓[/bold green] Space forked successfully")
|
| 195 |
+
console.print(
|
| 196 |
+
f"[bold]Space URL:[/bold] https://huggingface.co/spaces/{new_repo_id}"
|
| 197 |
+
)
|
src/openenv/cli/commands/init.py
ADDED
|
@@ -0,0 +1,500 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Initialize a new OpenEnv environment."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
import random
|
| 6 |
+
import shutil
|
| 7 |
+
import subprocess
|
| 8 |
+
from importlib import resources
|
| 9 |
+
from pathlib import Path
|
| 10 |
+
from typing import Annotated, Dict, List, Tuple
|
| 11 |
+
|
| 12 |
+
import typer
|
| 13 |
+
|
| 14 |
+
from .._cli_utils import console
|
| 15 |
+
|
| 16 |
+
app = typer.Typer(help="Initialize a new OpenEnv environment")
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
def _snake_to_pascal(snake_str: str) -> str:
|
| 20 |
+
"""Convert snake_case to PascalCase (e.g., 'my_env' -> 'MyEnv')."""
|
| 21 |
+
return "".join(word.capitalize() for word in snake_str.split("_"))
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
def _get_env_prefix(env_name: str) -> str:
|
| 25 |
+
"""Extract the prefix for class names (e.g., 'my_env' -> 'My', 'test_env' -> 'Test')."""
|
| 26 |
+
# Remove trailing '_env' if present
|
| 27 |
+
if env_name.endswith("_env"):
|
| 28 |
+
base = env_name[:-4] # Remove '_env'
|
| 29 |
+
else:
|
| 30 |
+
base = env_name
|
| 31 |
+
|
| 32 |
+
# If empty or just one part, use the whole thing
|
| 33 |
+
if not base or "_" not in base:
|
| 34 |
+
return base.capitalize() if base else env_name.capitalize()
|
| 35 |
+
|
| 36 |
+
# PascalCase all parts except the last
|
| 37 |
+
parts = base.split("_")
|
| 38 |
+
return "".join(word.capitalize() for word in parts)
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
def _snake_to_camel(snake_str: str) -> str:
|
| 42 |
+
"""Convert snake_case to camelCase (e.g., 'my_env' -> 'myEnv')."""
|
| 43 |
+
parts = snake_str.split("_")
|
| 44 |
+
return parts[0] + "".join(word.capitalize() for word in parts[1:])
|
| 45 |
+
|
| 46 |
+
|
| 47 |
+
def _snake_to_title(snake_str: str) -> str:
|
| 48 |
+
"""Convert snake_case to Title Case (e.g., 'my_env' -> 'My Env')."""
|
| 49 |
+
return " ".join(word.capitalize() for word in snake_str.split("_"))
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
def _validate_env_name(name: str) -> str:
|
| 53 |
+
"""Validate environment name (must be valid Python identifier in snake_case)."""
|
| 54 |
+
if not name:
|
| 55 |
+
raise typer.BadParameter("Environment name cannot be empty")
|
| 56 |
+
|
| 57 |
+
# Check if it's a valid Python identifier
|
| 58 |
+
if not name.isidentifier():
|
| 59 |
+
raise typer.BadParameter(
|
| 60 |
+
f"Environment name '{name}' is not a valid Python identifier. Use snake_case (e.g., 'my_env', 'game_env')."
|
| 61 |
+
)
|
| 62 |
+
|
| 63 |
+
# Check if it starts with a number
|
| 64 |
+
if name[0].isdigit():
|
| 65 |
+
raise typer.BadParameter(
|
| 66 |
+
f"Environment name '{name}' cannot start with a number."
|
| 67 |
+
)
|
| 68 |
+
|
| 69 |
+
return name
|
| 70 |
+
|
| 71 |
+
|
| 72 |
+
def _get_random_hf_space_config() -> Dict[str, str]:
|
| 73 |
+
"""
|
| 74 |
+
Get random Hugging Face Space configuration values.
|
| 75 |
+
|
| 76 |
+
Returns:
|
| 77 |
+
Dictionary with 'emoji', 'colorFrom', and 'colorTo' keys
|
| 78 |
+
"""
|
| 79 |
+
# Valid emojis (emoji-only characters)
|
| 80 |
+
emojis = [
|
| 81 |
+
"🎮",
|
| 82 |
+
"🎯",
|
| 83 |
+
"🚀",
|
| 84 |
+
"🌟",
|
| 85 |
+
"🎨",
|
| 86 |
+
"🎪",
|
| 87 |
+
"🎭",
|
| 88 |
+
"🎬",
|
| 89 |
+
"🎤",
|
| 90 |
+
"🎧",
|
| 91 |
+
"🎵",
|
| 92 |
+
"🎶",
|
| 93 |
+
"🎸",
|
| 94 |
+
"🎹",
|
| 95 |
+
"🥁",
|
| 96 |
+
"🎺",
|
| 97 |
+
"🎻",
|
| 98 |
+
"🎼",
|
| 99 |
+
"🎯",
|
| 100 |
+
"🎲",
|
| 101 |
+
"🎳",
|
| 102 |
+
"🎰",
|
| 103 |
+
"🎴",
|
| 104 |
+
"🃏",
|
| 105 |
+
"🀄",
|
| 106 |
+
"🎴",
|
| 107 |
+
"🎨",
|
| 108 |
+
"🖼️",
|
| 109 |
+
"🎬",
|
| 110 |
+
"🎭",
|
| 111 |
+
"🎪",
|
| 112 |
+
"🎤",
|
| 113 |
+
"🎧",
|
| 114 |
+
"🎵",
|
| 115 |
+
"🎶",
|
| 116 |
+
"🎸",
|
| 117 |
+
"🎹",
|
| 118 |
+
"🎺",
|
| 119 |
+
"🎻",
|
| 120 |
+
"🥁",
|
| 121 |
+
"🎯",
|
| 122 |
+
"🎲",
|
| 123 |
+
"🎳",
|
| 124 |
+
"🎰",
|
| 125 |
+
"🏀",
|
| 126 |
+
"⚽",
|
| 127 |
+
"🏈",
|
| 128 |
+
"⚾",
|
| 129 |
+
"🎾",
|
| 130 |
+
"🏐",
|
| 131 |
+
"🏉",
|
| 132 |
+
"🎱",
|
| 133 |
+
"🏓",
|
| 134 |
+
"🏸",
|
| 135 |
+
"🥅",
|
| 136 |
+
"🏒",
|
| 137 |
+
"🏑",
|
| 138 |
+
"🏏",
|
| 139 |
+
"⛳",
|
| 140 |
+
"🏹",
|
| 141 |
+
"🎣",
|
| 142 |
+
"🥊",
|
| 143 |
+
"🥋",
|
| 144 |
+
"🎽",
|
| 145 |
+
"🏅",
|
| 146 |
+
"🎖️",
|
| 147 |
+
"🏆",
|
| 148 |
+
"🥇",
|
| 149 |
+
"🥈",
|
| 150 |
+
"🥉",
|
| 151 |
+
"🔊",
|
| 152 |
+
"🔉",
|
| 153 |
+
"🔈",
|
| 154 |
+
"🔇",
|
| 155 |
+
"📢",
|
| 156 |
+
"📣",
|
| 157 |
+
"📯",
|
| 158 |
+
"🔔",
|
| 159 |
+
"🔕",
|
| 160 |
+
"📻",
|
| 161 |
+
"📡",
|
| 162 |
+
"💻",
|
| 163 |
+
"🖥️",
|
| 164 |
+
"🖨️",
|
| 165 |
+
"⌨️",
|
| 166 |
+
"🖱️",
|
| 167 |
+
"🖲️",
|
| 168 |
+
"🕹️",
|
| 169 |
+
"🗜️",
|
| 170 |
+
"💾",
|
| 171 |
+
"💿",
|
| 172 |
+
"📀",
|
| 173 |
+
"📼",
|
| 174 |
+
"📷",
|
| 175 |
+
"📸",
|
| 176 |
+
"📹",
|
| 177 |
+
"🎥",
|
| 178 |
+
"📽️",
|
| 179 |
+
"🎞️",
|
| 180 |
+
"📞",
|
| 181 |
+
"☎️",
|
| 182 |
+
"📟",
|
| 183 |
+
"📠",
|
| 184 |
+
"📺",
|
| 185 |
+
"📻",
|
| 186 |
+
"🎙️",
|
| 187 |
+
"🎚️",
|
| 188 |
+
"🎛️",
|
| 189 |
+
"⏱️",
|
| 190 |
+
"⏲️",
|
| 191 |
+
"⏰",
|
| 192 |
+
"🕰️",
|
| 193 |
+
"⌚",
|
| 194 |
+
"📱",
|
| 195 |
+
"📲",
|
| 196 |
+
"💻",
|
| 197 |
+
"⌨️",
|
| 198 |
+
"🖥️",
|
| 199 |
+
"🖨️",
|
| 200 |
+
"🖱️",
|
| 201 |
+
]
|
| 202 |
+
|
| 203 |
+
# Valid colors from HF Spaces config reference
|
| 204 |
+
colors = ["red", "yellow", "green", "blue", "indigo", "purple", "pink", "gray"]
|
| 205 |
+
|
| 206 |
+
return {
|
| 207 |
+
"emoji": random.choice(emojis),
|
| 208 |
+
"colorFrom": random.choice(colors),
|
| 209 |
+
"colorTo": random.choice(colors),
|
| 210 |
+
}
|
| 211 |
+
|
| 212 |
+
|
| 213 |
+
def _create_template_replacements(env_name: str) -> Dict[str, str]:
|
| 214 |
+
"""
|
| 215 |
+
Create comprehensive template replacement dictionary.
|
| 216 |
+
|
| 217 |
+
Supports all naming conventions:
|
| 218 |
+
- PascalCase for class names
|
| 219 |
+
- camelCase for variable names
|
| 220 |
+
- snake_case for module names, file paths
|
| 221 |
+
"""
|
| 222 |
+
env_prefix = _get_env_prefix(env_name)
|
| 223 |
+
env_camel = _snake_to_camel(env_name)
|
| 224 |
+
env_title = _snake_to_title(env_name)
|
| 225 |
+
|
| 226 |
+
# Get random HF Space config values
|
| 227 |
+
hf_config = _get_random_hf_space_config()
|
| 228 |
+
|
| 229 |
+
replacements = {
|
| 230 |
+
# Template placeholders (MUST come first - full class names before partial)
|
| 231 |
+
"__ENV_CLASS_NAME__Environment": f"{env_prefix}Environment",
|
| 232 |
+
"__ENV_CLASS_NAME__Action": f"{env_prefix}Action",
|
| 233 |
+
"__ENV_CLASS_NAME__Observation": f"{env_prefix}Observation",
|
| 234 |
+
"__ENV_CLASS_NAME__Env": f"{env_prefix}Env",
|
| 235 |
+
# Template placeholders (partial - must come after full replacements)
|
| 236 |
+
"__ENV_NAME__": env_name,
|
| 237 |
+
"__ENV_CLASS_NAME__": env_prefix, # Use prefix, not full PascalCase
|
| 238 |
+
"__ENV_TITLE_NAME__": env_title,
|
| 239 |
+
"__ENV_CAMEL_NAME__": env_camel,
|
| 240 |
+
# Hugging Face Space config placeholders
|
| 241 |
+
"__HF_EMOJI__": hf_config["emoji"],
|
| 242 |
+
"__HF_COLOR_FROM__": hf_config["colorFrom"],
|
| 243 |
+
"__HF_COLOR_TO__": hf_config["colorTo"],
|
| 244 |
+
}
|
| 245 |
+
|
| 246 |
+
return replacements
|
| 247 |
+
|
| 248 |
+
|
| 249 |
+
def _replace_in_content(content: str, replacements: Dict[str, str]) -> str:
|
| 250 |
+
"""Replace all occurrences in content using case-sensitive replacements."""
|
| 251 |
+
result = content
|
| 252 |
+
# Sort by length (longest first) to avoid partial replacements
|
| 253 |
+
for old, new in sorted(replacements.items(), key=lambda x: len(x[0]), reverse=True):
|
| 254 |
+
result = result.replace(old, new)
|
| 255 |
+
return result
|
| 256 |
+
|
| 257 |
+
|
| 258 |
+
def _should_rename_file(filename: str, env_name: str) -> Tuple[bool, str]:
|
| 259 |
+
"""
|
| 260 |
+
Check if a file should be renamed and return the new name.
|
| 261 |
+
|
| 262 |
+
Handles template placeholders in filenames like:
|
| 263 |
+
- `__ENV_NAME___environment.py` → `<env_name>_environment.py`
|
| 264 |
+
"""
|
| 265 |
+
# Check for template placeholder
|
| 266 |
+
if "__ENV_NAME__" in filename:
|
| 267 |
+
new_name = filename.replace("__ENV_NAME__", env_name)
|
| 268 |
+
return True, new_name
|
| 269 |
+
|
| 270 |
+
return False, filename
|
| 271 |
+
|
| 272 |
+
|
| 273 |
+
def _copy_and_template_file(
|
| 274 |
+
src_path: Path,
|
| 275 |
+
dest_path: Path,
|
| 276 |
+
replacements: Dict[str, str],
|
| 277 |
+
) -> None:
|
| 278 |
+
"""Copy a file and apply template replacements."""
|
| 279 |
+
dest_path.parent.mkdir(parents=True, exist_ok=True)
|
| 280 |
+
|
| 281 |
+
try:
|
| 282 |
+
# Read source file
|
| 283 |
+
content = src_path.read_bytes()
|
| 284 |
+
|
| 285 |
+
# Try to decode as text and apply replacements
|
| 286 |
+
try:
|
| 287 |
+
text = content.decode("utf-8")
|
| 288 |
+
# Normalize line endings to LF before applying replacements
|
| 289 |
+
text = text.replace("\r\n", "\n").replace("\r", "\n")
|
| 290 |
+
text = _replace_in_content(text, replacements)
|
| 291 |
+
dest_path.write_text(text, encoding="utf-8", newline="\n")
|
| 292 |
+
except UnicodeDecodeError:
|
| 293 |
+
# Binary file, just copy
|
| 294 |
+
dest_path.write_bytes(content)
|
| 295 |
+
except Exception as e:
|
| 296 |
+
raise RuntimeError(
|
| 297 |
+
f"Failed to copy template file {src_path} to {dest_path}: {e}"
|
| 298 |
+
) from e
|
| 299 |
+
|
| 300 |
+
|
| 301 |
+
def _copy_template_directory(
|
| 302 |
+
template_pkg: str,
|
| 303 |
+
template_dir: str,
|
| 304 |
+
dest_dir: Path,
|
| 305 |
+
replacements: Dict[str, str],
|
| 306 |
+
env_name: str,
|
| 307 |
+
) -> List[Path]:
|
| 308 |
+
"""Recursively copy template directory and apply replacements."""
|
| 309 |
+
created_files: List[Path] = []
|
| 310 |
+
|
| 311 |
+
# Get the package path using importlib.resources but avoid importing the template package
|
| 312 |
+
# We'll use the package's __file__ to get the directory path
|
| 313 |
+
import importlib
|
| 314 |
+
|
| 315 |
+
try:
|
| 316 |
+
# Import the parent package (not the template package itself)
|
| 317 |
+
if "." in template_pkg:
|
| 318 |
+
parent_pkg = ".".join(template_pkg.split(".")[:-1])
|
| 319 |
+
pkg = importlib.import_module(parent_pkg)
|
| 320 |
+
template_path = Path(pkg.__file__).parent / template_pkg.split(".")[-1]
|
| 321 |
+
else:
|
| 322 |
+
pkg = importlib.import_module(template_pkg.split(".")[0])
|
| 323 |
+
template_path = Path(pkg.__file__).parent / template_pkg.split(".")[-1]
|
| 324 |
+
except Exception:
|
| 325 |
+
# Fallback: try to use resources.files but handle import errors
|
| 326 |
+
try:
|
| 327 |
+
base = resources.files(template_pkg.split(".")[0])
|
| 328 |
+
template_path = base.joinpath(*template_pkg.split(".")[1:])
|
| 329 |
+
if not template_path.exists():
|
| 330 |
+
raise FileNotFoundError(f"Template directory not found: {template_pkg}")
|
| 331 |
+
except Exception as e:
|
| 332 |
+
raise FileNotFoundError(
|
| 333 |
+
f"Template directory not found: {template_pkg}"
|
| 334 |
+
) from e
|
| 335 |
+
|
| 336 |
+
if template_dir:
|
| 337 |
+
template_path = template_path / template_dir
|
| 338 |
+
|
| 339 |
+
if not template_path.exists() or not template_path.is_dir():
|
| 340 |
+
raise FileNotFoundError(
|
| 341 |
+
f"Template directory not found: {template_pkg}.{template_dir}"
|
| 342 |
+
)
|
| 343 |
+
|
| 344 |
+
# Walk through all files in template directory using Path
|
| 345 |
+
for item in template_path.rglob("*"):
|
| 346 |
+
if item.is_file():
|
| 347 |
+
rel_path = item.relative_to(template_path)
|
| 348 |
+
dest_path = dest_dir / rel_path
|
| 349 |
+
|
| 350 |
+
# Apply filename templating
|
| 351 |
+
should_rename, new_name = _should_rename_file(dest_path.name, env_name)
|
| 352 |
+
if should_rename:
|
| 353 |
+
dest_path = dest_path.parent / new_name
|
| 354 |
+
|
| 355 |
+
# Copy and apply replacements
|
| 356 |
+
_copy_and_template_file(item, dest_path, replacements)
|
| 357 |
+
created_files.append(dest_path)
|
| 358 |
+
|
| 359 |
+
return created_files
|
| 360 |
+
|
| 361 |
+
|
| 362 |
+
def _generate_uv_lock(env_dir: Path) -> bool:
|
| 363 |
+
"""Generate uv.lock from pyproject.toml using uv."""
|
| 364 |
+
pyproject_path = env_dir / "pyproject.toml"
|
| 365 |
+
|
| 366 |
+
if not pyproject_path.exists():
|
| 367 |
+
return False
|
| 368 |
+
|
| 369 |
+
try:
|
| 370 |
+
cmd = [
|
| 371 |
+
"uv",
|
| 372 |
+
"lock",
|
| 373 |
+
"--directory",
|
| 374 |
+
str(env_dir),
|
| 375 |
+
]
|
| 376 |
+
|
| 377 |
+
result = subprocess.run(cmd, capture_output=True, text=True, check=True)
|
| 378 |
+
|
| 379 |
+
if result.stdout:
|
| 380 |
+
console.print(result.stdout)
|
| 381 |
+
|
| 382 |
+
return True
|
| 383 |
+
|
| 384 |
+
except subprocess.CalledProcessError as e:
|
| 385 |
+
console.print(
|
| 386 |
+
f"[yellow]Warning: Could not generate uv.lock: {e.stderr}[/yellow]"
|
| 387 |
+
)
|
| 388 |
+
return False
|
| 389 |
+
except FileNotFoundError:
|
| 390 |
+
console.print(
|
| 391 |
+
"[yellow]Warning: 'uv' not found. Install it to generate uv.lock[/yellow]"
|
| 392 |
+
)
|
| 393 |
+
return False
|
| 394 |
+
|
| 395 |
+
|
| 396 |
+
@app.command()
|
| 397 |
+
def init(
|
| 398 |
+
env_name: Annotated[
|
| 399 |
+
str,
|
| 400 |
+
typer.Argument(
|
| 401 |
+
help="Name of the environment to create (snake_case, e.g., 'my_env')"
|
| 402 |
+
),
|
| 403 |
+
],
|
| 404 |
+
output_dir: Annotated[
|
| 405 |
+
str | None,
|
| 406 |
+
typer.Option(
|
| 407 |
+
"--output-dir",
|
| 408 |
+
"-o",
|
| 409 |
+
help="Output directory (defaults to current working directory)",
|
| 410 |
+
),
|
| 411 |
+
] = None,
|
| 412 |
+
) -> None:
|
| 413 |
+
"""
|
| 414 |
+
Initialize a new OpenEnv environment.
|
| 415 |
+
|
| 416 |
+
Creates a new directory with the environment name and generates all necessary
|
| 417 |
+
files based on the OpenEnv template structure.
|
| 418 |
+
|
| 419 |
+
Example:
|
| 420 |
+
$ openenv init my_game_env
|
| 421 |
+
$ openenv init my_env --output-dir /path/to/projects
|
| 422 |
+
"""
|
| 423 |
+
# Validate environment name
|
| 424 |
+
env_name = _validate_env_name(env_name)
|
| 425 |
+
|
| 426 |
+
# Determine output directory
|
| 427 |
+
base_dir = Path(output_dir).resolve() if output_dir else Path.cwd().resolve()
|
| 428 |
+
env_dir = base_dir / env_name
|
| 429 |
+
|
| 430 |
+
# Check if directory already exists
|
| 431 |
+
if env_dir.exists():
|
| 432 |
+
if env_dir.is_file():
|
| 433 |
+
raise typer.BadParameter(f"Path '{env_dir}' exists and is a file")
|
| 434 |
+
if any(env_dir.iterdir()):
|
| 435 |
+
raise typer.BadParameter(
|
| 436 |
+
f"Directory '{env_dir}' already exists and is not empty. "
|
| 437 |
+
"Please choose a different name or remove the existing directory."
|
| 438 |
+
)
|
| 439 |
+
|
| 440 |
+
try:
|
| 441 |
+
# Create template replacements
|
| 442 |
+
replacements = _create_template_replacements(env_name)
|
| 443 |
+
|
| 444 |
+
# Create environment directory
|
| 445 |
+
env_dir.mkdir(parents=True, exist_ok=True)
|
| 446 |
+
|
| 447 |
+
console.print(
|
| 448 |
+
f"[bold cyan]Creating OpenEnv environment '{env_name}'...[/bold cyan]"
|
| 449 |
+
)
|
| 450 |
+
|
| 451 |
+
# Copy template files from template structure
|
| 452 |
+
template_pkg = "openenv.cli.templates.openenv_env"
|
| 453 |
+
created_files = _copy_template_directory(
|
| 454 |
+
template_pkg,
|
| 455 |
+
"",
|
| 456 |
+
env_dir,
|
| 457 |
+
replacements,
|
| 458 |
+
env_name,
|
| 459 |
+
)
|
| 460 |
+
|
| 461 |
+
console.print(f"[bold green]✓[/bold green] Created {len(created_files)} files")
|
| 462 |
+
|
| 463 |
+
# Generate uv.lock
|
| 464 |
+
console.print("\n[bold]Generating uv.lock...[/bold]")
|
| 465 |
+
if _generate_uv_lock(env_dir):
|
| 466 |
+
console.print("[green]✓[/green] Generated uv.lock")
|
| 467 |
+
else:
|
| 468 |
+
console.print("[yellow]⚠[/yellow] Could not generate uv.lock automatically")
|
| 469 |
+
console.print(" You can generate it manually with:")
|
| 470 |
+
console.print(f" cd {env_dir} && uv lock")
|
| 471 |
+
|
| 472 |
+
console.print(
|
| 473 |
+
f"\n[bold green]Environment created successfully at: {env_dir}[/bold green]"
|
| 474 |
+
)
|
| 475 |
+
console.print("\n[bold]Next steps:[/bold]")
|
| 476 |
+
console.print(f" cd {env_dir}")
|
| 477 |
+
console.print(
|
| 478 |
+
f" # Edit your environment implementation in server/{env_name}_environment.py"
|
| 479 |
+
)
|
| 480 |
+
console.print(" # Edit your models in models.py")
|
| 481 |
+
console.print(" # Install dependencies: uv sync")
|
| 482 |
+
console.print("\n # To integrate into OpenEnv repo:")
|
| 483 |
+
console.print(f" # 1. Copy this directory to <repo_root>/envs/{env_name}_env")
|
| 484 |
+
console.print(
|
| 485 |
+
f" # 2. Build from repo root: docker build -t {env_name}_env:latest -f envs/{env_name}_env/server/Dockerfile ."
|
| 486 |
+
)
|
| 487 |
+
console.print(
|
| 488 |
+
f" # 3. Run your image: docker run -p 8000:8000 {env_name}_env:latest"
|
| 489 |
+
)
|
| 490 |
+
|
| 491 |
+
except Exception as e:
|
| 492 |
+
# Cleanup on error
|
| 493 |
+
if env_dir.exists() and env_dir.is_dir():
|
| 494 |
+
try:
|
| 495 |
+
shutil.rmtree(env_dir)
|
| 496 |
+
except Exception:
|
| 497 |
+
pass
|
| 498 |
+
|
| 499 |
+
console.print(f"[bold red]Error:[/bold red] {e}")
|
| 500 |
+
raise typer.Exit(1) from e
|
src/openenv/cli/commands/push.py
ADDED
|
@@ -0,0 +1,718 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Meta Platforms, Inc. and affiliates.
|
| 2 |
+
# All rights reserved.
|
| 3 |
+
#
|
| 4 |
+
# This source code is licensed under the BSD-style license found in the
|
| 5 |
+
# LICENSE file in the root directory of this source tree.
|
| 6 |
+
|
| 7 |
+
"""Push an OpenEnv environment to Hugging Face Spaces."""
|
| 8 |
+
|
| 9 |
+
from __future__ import annotations
|
| 10 |
+
|
| 11 |
+
import shutil
|
| 12 |
+
import sys
|
| 13 |
+
import tempfile
|
| 14 |
+
from fnmatch import fnmatch
|
| 15 |
+
from pathlib import Path
|
| 16 |
+
from typing import Annotated
|
| 17 |
+
|
| 18 |
+
import typer
|
| 19 |
+
import yaml
|
| 20 |
+
from huggingface_hub import HfApi, login, whoami
|
| 21 |
+
|
| 22 |
+
from .._cli_utils import console, validate_env_structure
|
| 23 |
+
|
| 24 |
+
app = typer.Typer(help="Push an OpenEnv environment to Hugging Face Spaces")
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
DEFAULT_PUSH_IGNORE_PATTERNS = [".*", "__pycache__", "*.pyc"]
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
def _path_matches_pattern(relative_path: Path, pattern: str) -> bool:
|
| 31 |
+
"""Return True if a relative path matches an exclude pattern."""
|
| 32 |
+
normalized_pattern = pattern.strip()
|
| 33 |
+
if normalized_pattern.startswith("!"):
|
| 34 |
+
return False
|
| 35 |
+
|
| 36 |
+
while normalized_pattern.startswith("./"):
|
| 37 |
+
normalized_pattern = normalized_pattern[2:]
|
| 38 |
+
|
| 39 |
+
if normalized_pattern.startswith("/"):
|
| 40 |
+
normalized_pattern = normalized_pattern[1:]
|
| 41 |
+
|
| 42 |
+
if not normalized_pattern:
|
| 43 |
+
return False
|
| 44 |
+
|
| 45 |
+
posix_path = relative_path.as_posix()
|
| 46 |
+
pattern_candidates = [normalized_pattern]
|
| 47 |
+
if normalized_pattern.startswith("**/"):
|
| 48 |
+
# Gitignore-style "**/" can also match directly at the root.
|
| 49 |
+
pattern_candidates.append(normalized_pattern[3:])
|
| 50 |
+
|
| 51 |
+
# Support directory patterns such as "artifacts/" and "**/outputs/".
|
| 52 |
+
if normalized_pattern.endswith("/"):
|
| 53 |
+
dir_pattern_candidates: list[str] = []
|
| 54 |
+
for candidate in pattern_candidates:
|
| 55 |
+
base = candidate.rstrip("/")
|
| 56 |
+
if not base:
|
| 57 |
+
continue
|
| 58 |
+
dir_pattern_candidates.extend([base, f"{base}/*"])
|
| 59 |
+
|
| 60 |
+
return any(
|
| 61 |
+
fnmatch(posix_path, candidate) for candidate in dir_pattern_candidates
|
| 62 |
+
)
|
| 63 |
+
|
| 64 |
+
# Match both full relative path and basename for convenience.
|
| 65 |
+
return any(
|
| 66 |
+
fnmatch(posix_path, candidate) for candidate in pattern_candidates
|
| 67 |
+
) or any(fnmatch(relative_path.name, candidate) for candidate in pattern_candidates)
|
| 68 |
+
|
| 69 |
+
|
| 70 |
+
def _should_exclude_path(relative_path: Path, ignore_patterns: list[str]) -> bool:
|
| 71 |
+
"""Return True when the path should be excluded from staging/upload."""
|
| 72 |
+
return any(
|
| 73 |
+
_path_matches_pattern(relative_path, pattern) for pattern in ignore_patterns
|
| 74 |
+
)
|
| 75 |
+
|
| 76 |
+
|
| 77 |
+
def _read_ignore_file(ignore_path: Path) -> tuple[list[str], int]:
|
| 78 |
+
"""Read ignore patterns from a file and return (patterns, ignored_negations)."""
|
| 79 |
+
patterns: list[str] = []
|
| 80 |
+
ignored_negations = 0
|
| 81 |
+
|
| 82 |
+
for line in ignore_path.read_text().splitlines():
|
| 83 |
+
stripped = line.strip()
|
| 84 |
+
if not stripped or stripped.startswith("#"):
|
| 85 |
+
continue
|
| 86 |
+
if stripped.startswith("!"):
|
| 87 |
+
ignored_negations += 1
|
| 88 |
+
continue
|
| 89 |
+
patterns.append(stripped)
|
| 90 |
+
|
| 91 |
+
return patterns, ignored_negations
|
| 92 |
+
|
| 93 |
+
|
| 94 |
+
def _load_ignore_patterns(env_dir: Path, exclude_file: str | None) -> list[str]:
|
| 95 |
+
"""Load ignore patterns from defaults and an optional ignore file."""
|
| 96 |
+
patterns = list(DEFAULT_PUSH_IGNORE_PATTERNS)
|
| 97 |
+
ignored_negations = 0
|
| 98 |
+
|
| 99 |
+
def _merge_ignore_file(ignore_path: Path, *, source_label: str) -> None:
|
| 100 |
+
nonlocal ignored_negations
|
| 101 |
+
file_patterns, skipped_negations = _read_ignore_file(ignore_path)
|
| 102 |
+
patterns.extend(file_patterns)
|
| 103 |
+
ignored_negations += skipped_negations
|
| 104 |
+
console.print(
|
| 105 |
+
f"[bold green]✓[/bold green] Loaded {len(file_patterns)} ignore patterns from {source_label}: {ignore_path}"
|
| 106 |
+
)
|
| 107 |
+
|
| 108 |
+
# Optional source: explicit exclude file from CLI.
|
| 109 |
+
if exclude_file:
|
| 110 |
+
ignore_path = Path(exclude_file)
|
| 111 |
+
if not ignore_path.is_absolute():
|
| 112 |
+
ignore_path = env_dir / ignore_path
|
| 113 |
+
ignore_path = ignore_path.resolve()
|
| 114 |
+
|
| 115 |
+
if not ignore_path.exists() or not ignore_path.is_file():
|
| 116 |
+
raise typer.BadParameter(
|
| 117 |
+
f"Exclude file not found or not a file: {ignore_path}"
|
| 118 |
+
)
|
| 119 |
+
|
| 120 |
+
_merge_ignore_file(ignore_path, source_label="--exclude")
|
| 121 |
+
|
| 122 |
+
# Keep stable order while removing duplicates.
|
| 123 |
+
patterns = list(dict.fromkeys(patterns))
|
| 124 |
+
|
| 125 |
+
if ignored_negations > 0:
|
| 126 |
+
console.print(
|
| 127 |
+
f"[bold yellow]⚠[/bold yellow] Skipped {ignored_negations} negated ignore patterns ('!') because negation is not supported for push excludes"
|
| 128 |
+
)
|
| 129 |
+
|
| 130 |
+
return patterns
|
| 131 |
+
|
| 132 |
+
|
| 133 |
+
def _copytree_ignore_factory(env_dir: Path, ignore_patterns: list[str]):
|
| 134 |
+
"""Build a shutil.copytree ignore callback from path-based patterns."""
|
| 135 |
+
|
| 136 |
+
def _ignore(path: str, names: list[str]) -> set[str]:
|
| 137 |
+
current_dir = Path(path)
|
| 138 |
+
ignored: set[str] = set()
|
| 139 |
+
|
| 140 |
+
for name in names:
|
| 141 |
+
candidate = current_dir / name
|
| 142 |
+
try:
|
| 143 |
+
relative_path = candidate.relative_to(env_dir)
|
| 144 |
+
except ValueError:
|
| 145 |
+
# candidate is not under env_dir (e.g. symlink or
|
| 146 |
+
# copytree root differs from env_dir); skip filtering.
|
| 147 |
+
continue
|
| 148 |
+
if _should_exclude_path(relative_path, ignore_patterns):
|
| 149 |
+
ignored.add(name)
|
| 150 |
+
|
| 151 |
+
return ignored
|
| 152 |
+
|
| 153 |
+
return _ignore
|
| 154 |
+
|
| 155 |
+
|
| 156 |
+
def _validate_openenv_directory(directory: Path) -> tuple[str, dict]:
|
| 157 |
+
"""
|
| 158 |
+
Validate that the directory is an OpenEnv environment.
|
| 159 |
+
|
| 160 |
+
Returns:
|
| 161 |
+
Tuple of (env_name, manifest_data)
|
| 162 |
+
"""
|
| 163 |
+
# Use the comprehensive validation function
|
| 164 |
+
try:
|
| 165 |
+
warnings = validate_env_structure(directory)
|
| 166 |
+
for warning in warnings:
|
| 167 |
+
console.print(f"[bold yellow]⚠[/bold yellow] {warning}")
|
| 168 |
+
except FileNotFoundError as e:
|
| 169 |
+
raise typer.BadParameter(f"Invalid OpenEnv environment structure: {e}") from e
|
| 170 |
+
|
| 171 |
+
# Load and validate manifest
|
| 172 |
+
manifest_path = directory / "openenv.yaml"
|
| 173 |
+
try:
|
| 174 |
+
with open(manifest_path, "r") as f:
|
| 175 |
+
manifest = yaml.safe_load(f)
|
| 176 |
+
except Exception as e:
|
| 177 |
+
raise typer.BadParameter(f"Failed to parse openenv.yaml: {e}") from e
|
| 178 |
+
|
| 179 |
+
if not isinstance(manifest, dict):
|
| 180 |
+
raise typer.BadParameter("openenv.yaml must be a YAML dictionary")
|
| 181 |
+
|
| 182 |
+
env_name = manifest.get("name")
|
| 183 |
+
if not env_name:
|
| 184 |
+
raise typer.BadParameter("openenv.yaml must contain a 'name' field")
|
| 185 |
+
|
| 186 |
+
return env_name, manifest
|
| 187 |
+
|
| 188 |
+
|
| 189 |
+
def _ensure_hf_authenticated() -> str:
|
| 190 |
+
"""
|
| 191 |
+
Ensure user is authenticated with Hugging Face.
|
| 192 |
+
|
| 193 |
+
Returns:
|
| 194 |
+
Username of authenticated user
|
| 195 |
+
"""
|
| 196 |
+
try:
|
| 197 |
+
# Try to get current user
|
| 198 |
+
user_info = whoami()
|
| 199 |
+
# Handle both dict and object return types
|
| 200 |
+
if isinstance(user_info, dict):
|
| 201 |
+
username = (
|
| 202 |
+
user_info.get("name")
|
| 203 |
+
or user_info.get("fullname")
|
| 204 |
+
or user_info.get("username")
|
| 205 |
+
)
|
| 206 |
+
else:
|
| 207 |
+
# If it's an object, try to get name attribute
|
| 208 |
+
username = (
|
| 209 |
+
getattr(user_info, "name", None)
|
| 210 |
+
or getattr(user_info, "fullname", None)
|
| 211 |
+
or getattr(user_info, "username", None)
|
| 212 |
+
)
|
| 213 |
+
|
| 214 |
+
if not username:
|
| 215 |
+
raise ValueError("Could not extract username from whoami response")
|
| 216 |
+
|
| 217 |
+
console.print(f"[bold green]✓[/bold green] Authenticated as: {username}")
|
| 218 |
+
return username
|
| 219 |
+
except Exception:
|
| 220 |
+
# Not authenticated, prompt for login
|
| 221 |
+
console.print(
|
| 222 |
+
"[bold yellow]Not authenticated with Hugging Face. Please login...[/bold yellow]"
|
| 223 |
+
)
|
| 224 |
+
|
| 225 |
+
try:
|
| 226 |
+
login()
|
| 227 |
+
# Verify login worked
|
| 228 |
+
user_info = whoami()
|
| 229 |
+
# Handle both dict and object return types
|
| 230 |
+
if isinstance(user_info, dict):
|
| 231 |
+
username = (
|
| 232 |
+
user_info.get("name")
|
| 233 |
+
or user_info.get("fullname")
|
| 234 |
+
or user_info.get("username")
|
| 235 |
+
)
|
| 236 |
+
else:
|
| 237 |
+
username = (
|
| 238 |
+
getattr(user_info, "name", None)
|
| 239 |
+
or getattr(user_info, "fullname", None)
|
| 240 |
+
or getattr(user_info, "username", None)
|
| 241 |
+
)
|
| 242 |
+
|
| 243 |
+
if not username:
|
| 244 |
+
raise ValueError("Could not extract username from whoami response")
|
| 245 |
+
|
| 246 |
+
console.print(f"[bold green]✓[/bold green] Authenticated as: {username}")
|
| 247 |
+
return username
|
| 248 |
+
except Exception as e:
|
| 249 |
+
raise typer.BadParameter(
|
| 250 |
+
f"Hugging Face authentication failed: {e}. Please run login manually."
|
| 251 |
+
) from e
|
| 252 |
+
|
| 253 |
+
|
| 254 |
+
def _prepare_staging_directory(
|
| 255 |
+
env_dir: Path,
|
| 256 |
+
env_name: str,
|
| 257 |
+
staging_dir: Path,
|
| 258 |
+
ignore_patterns: list[str],
|
| 259 |
+
base_image: str | None = None,
|
| 260 |
+
enable_interface: bool = True,
|
| 261 |
+
) -> None:
|
| 262 |
+
"""
|
| 263 |
+
Prepare files for deployment.
|
| 264 |
+
|
| 265 |
+
This includes:
|
| 266 |
+
- Copying necessary files
|
| 267 |
+
- Modifying Dockerfile to optionally enable web interface and update base image
|
| 268 |
+
- Ensuring README has proper HF frontmatter (if interface enabled)
|
| 269 |
+
"""
|
| 270 |
+
# Create staging directory structure
|
| 271 |
+
staging_dir.mkdir(parents=True, exist_ok=True)
|
| 272 |
+
|
| 273 |
+
# Copy all files from env directory
|
| 274 |
+
copy_ignore = _copytree_ignore_factory(env_dir, ignore_patterns)
|
| 275 |
+
for item in env_dir.iterdir():
|
| 276 |
+
relative_path = item.relative_to(env_dir)
|
| 277 |
+
if _should_exclude_path(relative_path, ignore_patterns):
|
| 278 |
+
continue
|
| 279 |
+
|
| 280 |
+
dest = staging_dir / item.name
|
| 281 |
+
if item.is_dir():
|
| 282 |
+
shutil.copytree(item, dest, dirs_exist_ok=True, ignore=copy_ignore)
|
| 283 |
+
else:
|
| 284 |
+
shutil.copy2(item, dest)
|
| 285 |
+
|
| 286 |
+
# Dockerfile must be at repo root for Hugging Face. Prefer root if present
|
| 287 |
+
# (it was copied there); otherwise move server/Dockerfile to root.
|
| 288 |
+
dockerfile_server_path = staging_dir / "server" / "Dockerfile"
|
| 289 |
+
dockerfile_root_path = staging_dir / "Dockerfile"
|
| 290 |
+
dockerfile_path: Path | None = None
|
| 291 |
+
|
| 292 |
+
if dockerfile_root_path.exists():
|
| 293 |
+
dockerfile_path = dockerfile_root_path
|
| 294 |
+
elif dockerfile_server_path.exists():
|
| 295 |
+
dockerfile_server_path.rename(dockerfile_root_path)
|
| 296 |
+
console.print(
|
| 297 |
+
"[bold cyan]Moved Dockerfile to repository root for deployment[/bold cyan]"
|
| 298 |
+
)
|
| 299 |
+
dockerfile_path = dockerfile_root_path
|
| 300 |
+
|
| 301 |
+
# Modify Dockerfile to optionally enable web interface and update base image
|
| 302 |
+
if dockerfile_path and dockerfile_path.exists():
|
| 303 |
+
dockerfile_content = dockerfile_path.read_text()
|
| 304 |
+
lines = dockerfile_content.split("\n")
|
| 305 |
+
new_lines = []
|
| 306 |
+
cmd_found = False
|
| 307 |
+
base_image_updated = False
|
| 308 |
+
web_interface_env_exists = "ENABLE_WEB_INTERFACE" in dockerfile_content
|
| 309 |
+
last_instruction = None
|
| 310 |
+
|
| 311 |
+
for line in lines:
|
| 312 |
+
stripped = line.strip()
|
| 313 |
+
token = stripped.split(maxsplit=1)[0] if stripped else ""
|
| 314 |
+
current_instruction = token.upper()
|
| 315 |
+
|
| 316 |
+
is_healthcheck_continuation = last_instruction == "HEALTHCHECK"
|
| 317 |
+
|
| 318 |
+
# Update base image if specified
|
| 319 |
+
if base_image and stripped.startswith("FROM") and not base_image_updated:
|
| 320 |
+
new_lines.append(f"FROM {base_image}")
|
| 321 |
+
base_image_updated = True
|
| 322 |
+
last_instruction = "FROM"
|
| 323 |
+
continue
|
| 324 |
+
|
| 325 |
+
if (
|
| 326 |
+
stripped.startswith("CMD")
|
| 327 |
+
and not cmd_found
|
| 328 |
+
and not web_interface_env_exists
|
| 329 |
+
and enable_interface
|
| 330 |
+
and not is_healthcheck_continuation
|
| 331 |
+
):
|
| 332 |
+
new_lines.append("ENV ENABLE_WEB_INTERFACE=true")
|
| 333 |
+
cmd_found = True
|
| 334 |
+
|
| 335 |
+
new_lines.append(line)
|
| 336 |
+
|
| 337 |
+
if current_instruction:
|
| 338 |
+
last_instruction = current_instruction
|
| 339 |
+
|
| 340 |
+
if not cmd_found and not web_interface_env_exists and enable_interface:
|
| 341 |
+
new_lines.append("ENV ENABLE_WEB_INTERFACE=true")
|
| 342 |
+
|
| 343 |
+
if base_image and not base_image_updated:
|
| 344 |
+
new_lines.insert(0, f"FROM {base_image}")
|
| 345 |
+
|
| 346 |
+
dockerfile_path.write_text("\n".join(new_lines))
|
| 347 |
+
|
| 348 |
+
changes = []
|
| 349 |
+
if base_image and base_image_updated:
|
| 350 |
+
changes.append("updated base image")
|
| 351 |
+
if enable_interface and not web_interface_env_exists:
|
| 352 |
+
changes.append("enabled web interface")
|
| 353 |
+
if changes:
|
| 354 |
+
console.print(
|
| 355 |
+
f"[bold green]✓[/bold green] Updated Dockerfile: {', '.join(changes)}"
|
| 356 |
+
)
|
| 357 |
+
else:
|
| 358 |
+
console.print(
|
| 359 |
+
"[bold yellow]⚠[/bold yellow] No Dockerfile at server/ or repo root"
|
| 360 |
+
)
|
| 361 |
+
|
| 362 |
+
# Ensure README has proper HF frontmatter (only if interface enabled)
|
| 363 |
+
if enable_interface:
|
| 364 |
+
readme_path = staging_dir / "README.md"
|
| 365 |
+
if readme_path.exists():
|
| 366 |
+
readme_content = readme_path.read_text()
|
| 367 |
+
if "base_path: /web" not in readme_content:
|
| 368 |
+
# Check if frontmatter exists
|
| 369 |
+
if readme_content.startswith("---"):
|
| 370 |
+
# Add base_path to existing frontmatter
|
| 371 |
+
lines = readme_content.split("\n")
|
| 372 |
+
new_lines = []
|
| 373 |
+
_in_frontmatter = True
|
| 374 |
+
for i, line in enumerate(lines):
|
| 375 |
+
new_lines.append(line)
|
| 376 |
+
if line.strip() == "---" and i > 0:
|
| 377 |
+
# End of frontmatter, add base_path before this line
|
| 378 |
+
if "base_path:" not in "\n".join(new_lines):
|
| 379 |
+
new_lines.insert(-1, "base_path: /web")
|
| 380 |
+
_in_frontmatter = False
|
| 381 |
+
readme_path.write_text("\n".join(new_lines))
|
| 382 |
+
else:
|
| 383 |
+
# No frontmatter, add it
|
| 384 |
+
frontmatter = f"""---
|
| 385 |
+
title: {env_name.replace("_", " ").title()} Environment Server
|
| 386 |
+
emoji: 🔊
|
| 387 |
+
colorFrom: '#00C9FF'
|
| 388 |
+
colorTo: '#1B2845'
|
| 389 |
+
sdk: docker
|
| 390 |
+
pinned: false
|
| 391 |
+
app_port: 8000
|
| 392 |
+
base_path: /web
|
| 393 |
+
tags:
|
| 394 |
+
- openenv
|
| 395 |
+
---
|
| 396 |
+
|
| 397 |
+
"""
|
| 398 |
+
readme_path.write_text(frontmatter + readme_content)
|
| 399 |
+
console.print(
|
| 400 |
+
"[bold green]✓[/bold green] Updated README with HF Space frontmatter"
|
| 401 |
+
)
|
| 402 |
+
else:
|
| 403 |
+
console.print("[bold yellow]⚠[/bold yellow] No README.md found")
|
| 404 |
+
|
| 405 |
+
|
| 406 |
+
def _create_hf_space(
|
| 407 |
+
repo_id: str,
|
| 408 |
+
api: HfApi,
|
| 409 |
+
private: bool = False,
|
| 410 |
+
) -> None:
|
| 411 |
+
"""Create a Hugging Face Space if it doesn't exist."""
|
| 412 |
+
console.print(f"[bold cyan]Creating/verifying space: {repo_id}[/bold cyan]")
|
| 413 |
+
|
| 414 |
+
try:
|
| 415 |
+
api.create_repo(
|
| 416 |
+
repo_id=repo_id,
|
| 417 |
+
repo_type="space",
|
| 418 |
+
space_sdk="docker",
|
| 419 |
+
private=private,
|
| 420 |
+
exist_ok=True,
|
| 421 |
+
)
|
| 422 |
+
console.print(f"[bold green]✓[/bold green] Space {repo_id} is ready")
|
| 423 |
+
except Exception as e:
|
| 424 |
+
# Space might already exist, which is okay with exist_ok=True
|
| 425 |
+
# But if there's another error, log it
|
| 426 |
+
console.print(f"[bold yellow]⚠[/bold yellow] Space creation: {e}")
|
| 427 |
+
|
| 428 |
+
|
| 429 |
+
def _upload_to_hf_space(
|
| 430 |
+
repo_id: str,
|
| 431 |
+
staging_dir: Path,
|
| 432 |
+
api: HfApi,
|
| 433 |
+
ignore_patterns: list[str],
|
| 434 |
+
private: bool = False,
|
| 435 |
+
create_pr: bool = False,
|
| 436 |
+
commit_message: str | None = None,
|
| 437 |
+
) -> None:
|
| 438 |
+
"""Upload files to Hugging Face Space."""
|
| 439 |
+
if create_pr:
|
| 440 |
+
console.print(
|
| 441 |
+
f"[bold cyan]Uploading files to {repo_id} (will open a Pull Request)...[/bold cyan]"
|
| 442 |
+
)
|
| 443 |
+
else:
|
| 444 |
+
console.print(f"[bold cyan]Uploading files to {repo_id}...[/bold cyan]")
|
| 445 |
+
|
| 446 |
+
upload_kwargs: dict = {
|
| 447 |
+
"folder_path": str(staging_dir),
|
| 448 |
+
"repo_id": repo_id,
|
| 449 |
+
"repo_type": "space",
|
| 450 |
+
"create_pr": create_pr,
|
| 451 |
+
"ignore_patterns": ignore_patterns,
|
| 452 |
+
}
|
| 453 |
+
if commit_message:
|
| 454 |
+
upload_kwargs["commit_message"] = commit_message
|
| 455 |
+
|
| 456 |
+
try:
|
| 457 |
+
result = api.upload_folder(**upload_kwargs)
|
| 458 |
+
console.print("[bold green]✓[/bold green] Upload completed successfully")
|
| 459 |
+
if create_pr and result is not None and hasattr(result, "pr_url"):
|
| 460 |
+
console.print(f"[bold]Pull request:[/bold] {result.pr_url}")
|
| 461 |
+
console.print(
|
| 462 |
+
f"[bold]Space URL:[/bold] https://huggingface.co/spaces/{repo_id}"
|
| 463 |
+
)
|
| 464 |
+
except Exception as e:
|
| 465 |
+
console.print(f"[bold red]✗[/bold red] Upload failed: {e}")
|
| 466 |
+
raise typer.Exit(1) from e
|
| 467 |
+
|
| 468 |
+
|
| 469 |
+
@app.command()
|
| 470 |
+
def push(
|
| 471 |
+
directory: Annotated[
|
| 472 |
+
str | None,
|
| 473 |
+
typer.Argument(
|
| 474 |
+
help="Directory containing the OpenEnv environment (default: current directory)"
|
| 475 |
+
),
|
| 476 |
+
] = None,
|
| 477 |
+
repo_id: Annotated[
|
| 478 |
+
str | None,
|
| 479 |
+
typer.Option(
|
| 480 |
+
"--repo-id",
|
| 481 |
+
"-r",
|
| 482 |
+
help="Repository ID in format 'username/repo-name' (defaults to 'username/env-name' from openenv.yaml)",
|
| 483 |
+
),
|
| 484 |
+
] = None,
|
| 485 |
+
base_image: Annotated[
|
| 486 |
+
str | None,
|
| 487 |
+
typer.Option(
|
| 488 |
+
"--base-image",
|
| 489 |
+
"-b",
|
| 490 |
+
help="Base Docker image to use (overrides Dockerfile FROM)",
|
| 491 |
+
),
|
| 492 |
+
] = None,
|
| 493 |
+
interface: Annotated[
|
| 494 |
+
bool,
|
| 495 |
+
typer.Option(
|
| 496 |
+
"--interface",
|
| 497 |
+
help="Enable web interface (default: True if no registry specified)",
|
| 498 |
+
),
|
| 499 |
+
] = None,
|
| 500 |
+
no_interface: Annotated[
|
| 501 |
+
bool,
|
| 502 |
+
typer.Option(
|
| 503 |
+
"--no-interface",
|
| 504 |
+
help="Disable web interface",
|
| 505 |
+
),
|
| 506 |
+
] = False,
|
| 507 |
+
registry: Annotated[
|
| 508 |
+
str | None,
|
| 509 |
+
typer.Option(
|
| 510 |
+
"--registry",
|
| 511 |
+
help="Custom registry URL (e.g., docker.io/username). Disables web interface by default.",
|
| 512 |
+
),
|
| 513 |
+
] = None,
|
| 514 |
+
private: Annotated[
|
| 515 |
+
bool,
|
| 516 |
+
typer.Option(
|
| 517 |
+
"--private",
|
| 518 |
+
help="Deploy the space as private",
|
| 519 |
+
),
|
| 520 |
+
] = False,
|
| 521 |
+
create_pr: Annotated[
|
| 522 |
+
bool,
|
| 523 |
+
typer.Option(
|
| 524 |
+
"--create-pr",
|
| 525 |
+
help="Create a Pull Request instead of pushing to the default branch",
|
| 526 |
+
),
|
| 527 |
+
] = False,
|
| 528 |
+
exclude: Annotated[
|
| 529 |
+
str | None,
|
| 530 |
+
typer.Option(
|
| 531 |
+
"--exclude",
|
| 532 |
+
help="Optional additional ignore file with newline-separated glob patterns to exclude from Hugging Face uploads",
|
| 533 |
+
),
|
| 534 |
+
] = None,
|
| 535 |
+
) -> None:
|
| 536 |
+
"""
|
| 537 |
+
Push an OpenEnv environment to Hugging Face Spaces or a custom Docker registry.
|
| 538 |
+
|
| 539 |
+
This command:
|
| 540 |
+
1. Validates that the directory is an OpenEnv environment (openenv.yaml present)
|
| 541 |
+
2. Builds and pushes to Hugging Face Spaces or custom Docker registry
|
| 542 |
+
3. Optionally enables web interface for deployment
|
| 543 |
+
|
| 544 |
+
The web interface is enabled by default when pushing to HuggingFace Spaces,
|
| 545 |
+
but disabled by default when pushing to a custom Docker registry.
|
| 546 |
+
|
| 547 |
+
Examples:
|
| 548 |
+
# Push to HuggingFace Spaces from current directory (web interface enabled)
|
| 549 |
+
$ cd my_env
|
| 550 |
+
$ openenv push
|
| 551 |
+
|
| 552 |
+
# Push to HuggingFace repo and open a Pull Request
|
| 553 |
+
$ openenv push my-org/my-env --create-pr
|
| 554 |
+
$ openenv push --repo-id my-org/my-env --create-pr
|
| 555 |
+
|
| 556 |
+
# Push to HuggingFace without web interface
|
| 557 |
+
$ openenv push --no-interface
|
| 558 |
+
|
| 559 |
+
# Push to Docker Hub
|
| 560 |
+
$ openenv push --registry docker.io/myuser
|
| 561 |
+
|
| 562 |
+
# Push to GitHub Container Registry
|
| 563 |
+
$ openenv push --registry ghcr.io/myorg
|
| 564 |
+
|
| 565 |
+
# Push to custom registry with web interface
|
| 566 |
+
$ openenv push --registry myregistry.io/path1/path2 --interface
|
| 567 |
+
|
| 568 |
+
# Push to specific HuggingFace repo
|
| 569 |
+
$ openenv push --repo-id my-org/my-env
|
| 570 |
+
|
| 571 |
+
# Push privately with custom base image
|
| 572 |
+
$ openenv push --private --base-image ghcr.io/meta-pytorch/openenv-base:latest
|
| 573 |
+
"""
|
| 574 |
+
# Handle interface flag logic
|
| 575 |
+
if no_interface and interface:
|
| 576 |
+
console.print(
|
| 577 |
+
"[bold red]Error:[/bold red] Cannot specify both --interface and --no-interface",
|
| 578 |
+
file=sys.stderr,
|
| 579 |
+
)
|
| 580 |
+
raise typer.Exit(1)
|
| 581 |
+
|
| 582 |
+
# Determine if web interface should be enabled
|
| 583 |
+
if no_interface:
|
| 584 |
+
enable_interface = False
|
| 585 |
+
elif interface is not None:
|
| 586 |
+
enable_interface = interface
|
| 587 |
+
elif registry is not None:
|
| 588 |
+
# Custom registry: disable interface by default
|
| 589 |
+
enable_interface = False
|
| 590 |
+
else:
|
| 591 |
+
# HuggingFace: enable interface by default
|
| 592 |
+
enable_interface = True
|
| 593 |
+
|
| 594 |
+
# Determine directory
|
| 595 |
+
if directory:
|
| 596 |
+
env_dir = Path(directory).resolve()
|
| 597 |
+
else:
|
| 598 |
+
env_dir = Path.cwd().resolve()
|
| 599 |
+
|
| 600 |
+
if not env_dir.exists() or not env_dir.is_dir():
|
| 601 |
+
raise typer.BadParameter(f"Directory does not exist: {env_dir}")
|
| 602 |
+
|
| 603 |
+
# Check for openenv.yaml to confirm this is an environment directory
|
| 604 |
+
openenv_yaml = env_dir / "openenv.yaml"
|
| 605 |
+
if not openenv_yaml.exists():
|
| 606 |
+
console.print(
|
| 607 |
+
f"[bold red]Error:[/bold red] Not an OpenEnv environment directory (missing openenv.yaml): {env_dir}",
|
| 608 |
+
)
|
| 609 |
+
console.print(
|
| 610 |
+
"[yellow]Hint:[/yellow] Run this command from the environment root directory",
|
| 611 |
+
)
|
| 612 |
+
raise typer.Exit(1)
|
| 613 |
+
|
| 614 |
+
# Validate OpenEnv environment
|
| 615 |
+
console.print(
|
| 616 |
+
f"[bold cyan]Validating OpenEnv environment in {env_dir}...[/bold cyan]"
|
| 617 |
+
)
|
| 618 |
+
env_name, manifest = _validate_openenv_directory(env_dir)
|
| 619 |
+
console.print(f"[bold green]✓[/bold green] Found OpenEnv environment: {env_name}")
|
| 620 |
+
|
| 621 |
+
# Handle custom registry push
|
| 622 |
+
if registry:
|
| 623 |
+
console.print("[bold cyan]Preparing to push to custom registry...[/bold cyan]")
|
| 624 |
+
if enable_interface:
|
| 625 |
+
console.print("[bold cyan]Web interface will be enabled[/bold cyan]")
|
| 626 |
+
|
| 627 |
+
# Import build functions
|
| 628 |
+
from .build import _build_docker_image, _push_docker_image
|
| 629 |
+
|
| 630 |
+
# Prepare build args for custom registry deployment
|
| 631 |
+
build_args = {}
|
| 632 |
+
if enable_interface:
|
| 633 |
+
build_args["ENABLE_WEB_INTERFACE"] = "true"
|
| 634 |
+
|
| 635 |
+
# Build Docker image from the environment directory
|
| 636 |
+
tag = f"{registry}/{env_name}"
|
| 637 |
+
console.print(f"[bold cyan]Building Docker image: {tag}[/bold cyan]")
|
| 638 |
+
|
| 639 |
+
success = _build_docker_image(
|
| 640 |
+
env_path=env_dir,
|
| 641 |
+
tag=tag,
|
| 642 |
+
build_args=build_args if build_args else None,
|
| 643 |
+
)
|
| 644 |
+
|
| 645 |
+
if not success:
|
| 646 |
+
console.print("[bold red]✗ Docker build failed[/bold red]")
|
| 647 |
+
raise typer.Exit(1)
|
| 648 |
+
|
| 649 |
+
console.print("[bold green]✓ Docker build successful[/bold green]")
|
| 650 |
+
|
| 651 |
+
# Push to registry
|
| 652 |
+
console.print(f"[bold cyan]Pushing to registry: {registry}[/bold cyan]")
|
| 653 |
+
|
| 654 |
+
success = _push_docker_image(
|
| 655 |
+
tag, registry=None
|
| 656 |
+
) # Tag already includes registry
|
| 657 |
+
|
| 658 |
+
if not success:
|
| 659 |
+
console.print("[bold red]✗ Docker push failed[/bold red]")
|
| 660 |
+
raise typer.Exit(1)
|
| 661 |
+
|
| 662 |
+
console.print("\n[bold green]✓ Deployment complete![/bold green]")
|
| 663 |
+
console.print(f"[bold]Image:[/bold] {tag}")
|
| 664 |
+
return
|
| 665 |
+
|
| 666 |
+
ignore_patterns = _load_ignore_patterns(env_dir, exclude)
|
| 667 |
+
|
| 668 |
+
# Ensure authentication for HuggingFace
|
| 669 |
+
username = _ensure_hf_authenticated()
|
| 670 |
+
|
| 671 |
+
# Determine repo_id
|
| 672 |
+
if not repo_id:
|
| 673 |
+
repo_id = f"{username}/{env_name}"
|
| 674 |
+
|
| 675 |
+
# Validate repo_id format
|
| 676 |
+
if "/" not in repo_id or repo_id.count("/") != 1:
|
| 677 |
+
raise typer.BadParameter(
|
| 678 |
+
f"Invalid repo-id format: {repo_id}. Expected format: 'username/repo-name'"
|
| 679 |
+
)
|
| 680 |
+
|
| 681 |
+
# Initialize Hugging Face API
|
| 682 |
+
api = HfApi()
|
| 683 |
+
|
| 684 |
+
# Prepare staging directory
|
| 685 |
+
deployment_type = (
|
| 686 |
+
"with web interface" if enable_interface else "without web interface"
|
| 687 |
+
)
|
| 688 |
+
console.print(
|
| 689 |
+
f"[bold cyan]Preparing files for Hugging Face deployment ({deployment_type})...[/bold cyan]"
|
| 690 |
+
)
|
| 691 |
+
with tempfile.TemporaryDirectory() as tmpdir:
|
| 692 |
+
staging_dir = Path(tmpdir) / "staging"
|
| 693 |
+
_prepare_staging_directory(
|
| 694 |
+
env_dir,
|
| 695 |
+
env_name,
|
| 696 |
+
staging_dir,
|
| 697 |
+
ignore_patterns=ignore_patterns,
|
| 698 |
+
base_image=base_image,
|
| 699 |
+
enable_interface=enable_interface,
|
| 700 |
+
)
|
| 701 |
+
|
| 702 |
+
# Create/verify space (no-op if exists; needed when pushing to own new repo)
|
| 703 |
+
if not create_pr:
|
| 704 |
+
_create_hf_space(repo_id, api, private=private)
|
| 705 |
+
# When create_pr we rely on upload_folder to create branch and PR
|
| 706 |
+
|
| 707 |
+
# Upload files
|
| 708 |
+
_upload_to_hf_space(
|
| 709 |
+
repo_id,
|
| 710 |
+
staging_dir,
|
| 711 |
+
api,
|
| 712 |
+
private=private,
|
| 713 |
+
create_pr=create_pr,
|
| 714 |
+
ignore_patterns=ignore_patterns,
|
| 715 |
+
)
|
| 716 |
+
|
| 717 |
+
console.print("\n[bold green]✓ Deployment complete![/bold green]")
|
| 718 |
+
console.print(f"Visit your space at: https://huggingface.co/spaces/{repo_id}")
|
src/openenv/cli/commands/serve.py
ADDED
|
@@ -0,0 +1,94 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Meta Platforms, Inc. and affiliates.
|
| 2 |
+
# All rights reserved.
|
| 3 |
+
#
|
| 4 |
+
# This source code is licensed under the BSD-style license found in the
|
| 5 |
+
# LICENSE file in the root directory of this source tree.
|
| 6 |
+
|
| 7 |
+
"""Serve OpenEnv environments locally (TO BE IMPLEMENTED)."""
|
| 8 |
+
|
| 9 |
+
from __future__ import annotations
|
| 10 |
+
|
| 11 |
+
from pathlib import Path
|
| 12 |
+
from typing import Annotated
|
| 13 |
+
|
| 14 |
+
import typer
|
| 15 |
+
|
| 16 |
+
from .._cli_utils import console
|
| 17 |
+
|
| 18 |
+
app = typer.Typer(help="Serve OpenEnv environments locally")
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
@app.command()
|
| 22 |
+
def serve(
|
| 23 |
+
env_path: Annotated[
|
| 24 |
+
str | None,
|
| 25 |
+
typer.Argument(
|
| 26 |
+
help="Path to the environment directory (default: current directory)"
|
| 27 |
+
),
|
| 28 |
+
] = None,
|
| 29 |
+
port: Annotated[
|
| 30 |
+
int,
|
| 31 |
+
typer.Option("--port", "-p", help="Port to serve on"),
|
| 32 |
+
] = 8000,
|
| 33 |
+
host: Annotated[
|
| 34 |
+
str,
|
| 35 |
+
typer.Option("--host", help="Host to bind to"),
|
| 36 |
+
] = "0.0.0.0",
|
| 37 |
+
reload: Annotated[
|
| 38 |
+
bool,
|
| 39 |
+
typer.Option("--reload", help="Enable auto-reload on code changes"),
|
| 40 |
+
] = False,
|
| 41 |
+
) -> None:
|
| 42 |
+
"""
|
| 43 |
+
Serve an OpenEnv environment locally.
|
| 44 |
+
|
| 45 |
+
TODO: This command is currently not implemented and has been deferred for later.
|
| 46 |
+
|
| 47 |
+
Planned functionality:
|
| 48 |
+
- Run environment server locally without Docker
|
| 49 |
+
- Support multiple deployment modes (local, notebook, cluster)
|
| 50 |
+
- Auto-reload for development
|
| 51 |
+
- Integration with environment's [project.scripts] entry point
|
| 52 |
+
|
| 53 |
+
For now, use Docker-based serving:
|
| 54 |
+
1. Build the environment: openenv build
|
| 55 |
+
2. Run the container: docker run -p 8000:8000 <image-name>
|
| 56 |
+
|
| 57 |
+
Or use uv directly:
|
| 58 |
+
uv run --project . server --port 8000
|
| 59 |
+
"""
|
| 60 |
+
console.print("[bold yellow]⚠ This command is not yet implemented[/bold yellow]\n")
|
| 61 |
+
|
| 62 |
+
console.print(
|
| 63 |
+
"The [bold cyan]openenv serve[/bold cyan] command has been deferred for later."
|
| 64 |
+
)
|
| 65 |
+
|
| 66 |
+
console.print("[bold]Alternative approaches:[/bold]\n")
|
| 67 |
+
|
| 68 |
+
console.print("[cyan]Option 1: Docker-based serving (recommended)[/cyan]")
|
| 69 |
+
console.print(" 1. Build the environment:")
|
| 70 |
+
console.print(" [dim]$ openenv build[/dim]")
|
| 71 |
+
console.print(" 2. Run the Docker container:")
|
| 72 |
+
console.print(
|
| 73 |
+
f" [dim]$ docker run -p {port}:{port} openenv-<env-name>:latest[/dim]\n"
|
| 74 |
+
)
|
| 75 |
+
|
| 76 |
+
console.print("[cyan]Option 2: Direct execution with uv[/cyan]")
|
| 77 |
+
|
| 78 |
+
# Determine environment path
|
| 79 |
+
if env_path is None:
|
| 80 |
+
env_path_obj = Path.cwd()
|
| 81 |
+
else:
|
| 82 |
+
env_path_obj = Path(env_path)
|
| 83 |
+
|
| 84 |
+
# Check for openenv.yaml
|
| 85 |
+
openenv_yaml = env_path_obj / "openenv.yaml"
|
| 86 |
+
if openenv_yaml.exists():
|
| 87 |
+
console.print(" From your environment directory:")
|
| 88 |
+
console.print(f" [dim]$ cd {env_path_obj}[/dim]")
|
| 89 |
+
console.print(f" [dim]$ uv run --project . server --port {port}[/dim]\n")
|
| 90 |
+
else:
|
| 91 |
+
console.print(" From an environment directory with pyproject.toml:")
|
| 92 |
+
console.print(f" [dim]$ uv run --project . server --port {port}[/dim]\n")
|
| 93 |
+
|
| 94 |
+
raise typer.Exit(0)
|
src/openenv/cli/commands/validate.py
ADDED
|
@@ -0,0 +1,108 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Meta Platforms, Inc. and affiliates.
|
| 2 |
+
# All rights reserved.
|
| 3 |
+
#
|
| 4 |
+
# This source code is licensed under the BSD-style license found in the
|
| 5 |
+
# LICENSE file in the root directory of this source tree.
|
| 6 |
+
|
| 7 |
+
"""
|
| 8 |
+
OpenEnv validate command.
|
| 9 |
+
|
| 10 |
+
This module provides the 'openenv validate' command to check if environments
|
| 11 |
+
are properly configured for multi-mode deployment.
|
| 12 |
+
"""
|
| 13 |
+
|
| 14 |
+
from pathlib import Path
|
| 15 |
+
|
| 16 |
+
import typer
|
| 17 |
+
|
| 18 |
+
from openenv.cli._validation import (
|
| 19 |
+
format_validation_report,
|
| 20 |
+
get_deployment_modes,
|
| 21 |
+
validate_multi_mode_deployment,
|
| 22 |
+
)
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
def validate(
|
| 26 |
+
env_path: str | None = typer.Argument(
|
| 27 |
+
None, help="Path to the environment directory (default: current directory)"
|
| 28 |
+
),
|
| 29 |
+
verbose: bool = typer.Option(
|
| 30 |
+
False, "--verbose", "-v", help="Show detailed information"
|
| 31 |
+
),
|
| 32 |
+
) -> None:
|
| 33 |
+
"""
|
| 34 |
+
Validate an environment for standardized structure and deployment readiness.
|
| 35 |
+
|
| 36 |
+
This command checks if an environment is properly configured with:
|
| 37 |
+
- Required files (pyproject.toml, openenv.yaml, server/app.py, etc.)
|
| 38 |
+
- Docker deployment support
|
| 39 |
+
- uv run server capability
|
| 40 |
+
- python -m module execution
|
| 41 |
+
|
| 42 |
+
Examples:
|
| 43 |
+
# Validate current directory (recommended)
|
| 44 |
+
$ cd my_env
|
| 45 |
+
$ openenv validate
|
| 46 |
+
|
| 47 |
+
# Validate with detailed output
|
| 48 |
+
$ openenv validate --verbose
|
| 49 |
+
|
| 50 |
+
# Validate specific environment
|
| 51 |
+
$ openenv validate envs/echo_env
|
| 52 |
+
"""
|
| 53 |
+
# Determine environment path (default to current directory)
|
| 54 |
+
if env_path is None:
|
| 55 |
+
env_path_obj = Path.cwd()
|
| 56 |
+
else:
|
| 57 |
+
env_path_obj = Path(env_path)
|
| 58 |
+
|
| 59 |
+
if not env_path_obj.exists():
|
| 60 |
+
typer.echo(f"Error: Path does not exist: {env_path_obj}", err=True)
|
| 61 |
+
raise typer.Exit(1)
|
| 62 |
+
|
| 63 |
+
if not env_path_obj.is_dir():
|
| 64 |
+
typer.echo(f"Error: Path is not a directory: {env_path_obj}", err=True)
|
| 65 |
+
raise typer.Exit(1)
|
| 66 |
+
|
| 67 |
+
# Check for openenv.yaml to confirm this is an environment directory
|
| 68 |
+
openenv_yaml = env_path_obj / "openenv.yaml"
|
| 69 |
+
if not openenv_yaml.exists():
|
| 70 |
+
typer.echo(
|
| 71 |
+
f"Error: Not an OpenEnv environment directory (missing openenv.yaml): {env_path_obj}",
|
| 72 |
+
err=True,
|
| 73 |
+
)
|
| 74 |
+
typer.echo(
|
| 75 |
+
"Hint: Run this command from the environment root directory or specify the path",
|
| 76 |
+
err=True,
|
| 77 |
+
)
|
| 78 |
+
raise typer.Exit(1)
|
| 79 |
+
|
| 80 |
+
env_name = env_path_obj.name
|
| 81 |
+
if env_name.endswith("_env"):
|
| 82 |
+
base_name = env_name[:-4]
|
| 83 |
+
else:
|
| 84 |
+
base_name = env_name
|
| 85 |
+
|
| 86 |
+
# Run validation
|
| 87 |
+
is_valid, issues = validate_multi_mode_deployment(env_path_obj)
|
| 88 |
+
|
| 89 |
+
# Show validation report
|
| 90 |
+
report = format_validation_report(base_name, is_valid, issues)
|
| 91 |
+
typer.echo(report)
|
| 92 |
+
|
| 93 |
+
# Show deployment modes if verbose
|
| 94 |
+
if verbose:
|
| 95 |
+
typer.echo("\nSupported deployment modes:")
|
| 96 |
+
modes = get_deployment_modes(env_path_obj)
|
| 97 |
+
for mode, supported in modes.items():
|
| 98 |
+
status = "[YES]" if supported else "[NO]"
|
| 99 |
+
typer.echo(f" {status} {mode}")
|
| 100 |
+
|
| 101 |
+
if is_valid:
|
| 102 |
+
typer.echo("\nUsage examples:")
|
| 103 |
+
typer.echo(f" cd {env_path_obj.name} && uv run server")
|
| 104 |
+
typer.echo(f" cd {env_path_obj.name} && openenv build")
|
| 105 |
+
typer.echo(f" cd {env_path_obj.name} && openenv push")
|
| 106 |
+
|
| 107 |
+
if not is_valid:
|
| 108 |
+
raise typer.Exit(1)
|
src/openenv/cli/templates/__init__.py
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Meta Platforms, Inc. and affiliates.
|
| 2 |
+
# All rights reserved.
|
| 3 |
+
#
|
| 4 |
+
# This source code is licensed under the BSD-style license found in the
|
| 5 |
+
# LICENSE file in the root directory of this source tree.
|
| 6 |
+
|
| 7 |
+
"""OpenEnv CLI templates package."""
|
src/openenv/cli/templates/openenv_env/.dockerignore
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
.venv
|
| 2 |
+
.git
|
| 3 |
+
.gitignore
|
| 4 |
+
.env
|
| 5 |
+
__pycache__/
|
| 6 |
+
*.pyc
|
| 7 |
+
*.pyo
|
| 8 |
+
*.pyd
|
| 9 |
+
*.pyw
|
| 10 |
+
*.pyz
|
| 11 |
+
*.pywz
|
| 12 |
+
*.pyzw
|
| 13 |
+
*.pyzwz
|
| 14 |
+
|
| 15 |
+
|
src/openenv/cli/templates/openenv_env/README.md
ADDED
|
@@ -0,0 +1,255 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
---
|
| 2 |
+
title: __ENV_TITLE_NAME__ Environment Server
|
| 3 |
+
emoji: __HF_EMOJI__
|
| 4 |
+
colorFrom: __HF_COLOR_FROM__
|
| 5 |
+
colorTo: __HF_COLOR_TO__
|
| 6 |
+
sdk: docker
|
| 7 |
+
pinned: false
|
| 8 |
+
app_port: 8000
|
| 9 |
+
base_path: /web
|
| 10 |
+
tags:
|
| 11 |
+
- openenv
|
| 12 |
+
---
|
| 13 |
+
|
| 14 |
+
# __ENV_TITLE_NAME__ Environment
|
| 15 |
+
|
| 16 |
+
A simple test environment that echoes back messages. Perfect for testing the env APIs as well as demonstrating environment usage patterns.
|
| 17 |
+
|
| 18 |
+
## Quick Start
|
| 19 |
+
|
| 20 |
+
The simplest way to use the __ENV_TITLE_NAME__ environment is through the `__ENV_CLASS_NAME__Env` class:
|
| 21 |
+
|
| 22 |
+
```python
|
| 23 |
+
from __ENV_NAME__ import __ENV_CLASS_NAME__Action, __ENV_CLASS_NAME__Env
|
| 24 |
+
|
| 25 |
+
try:
|
| 26 |
+
# Create environment from Docker image
|
| 27 |
+
__ENV_NAME__env = __ENV_CLASS_NAME__Env.from_docker_image("__ENV_NAME__-env:latest")
|
| 28 |
+
|
| 29 |
+
# Reset
|
| 30 |
+
result = __ENV_NAME__env.reset()
|
| 31 |
+
print(f"Reset: {result.observation.echoed_message}")
|
| 32 |
+
|
| 33 |
+
# Send multiple messages
|
| 34 |
+
messages = ["Hello, World!", "Testing echo", "Final message"]
|
| 35 |
+
|
| 36 |
+
for msg in messages:
|
| 37 |
+
result = __ENV_NAME__env.step(__ENV_CLASS_NAME__Action(message=msg))
|
| 38 |
+
print(f"Sent: '{msg}'")
|
| 39 |
+
print(f" → Echoed: '{result.observation.echoed_message}'")
|
| 40 |
+
print(f" → Length: {result.observation.message_length}")
|
| 41 |
+
print(f" → Reward: {result.reward}")
|
| 42 |
+
|
| 43 |
+
finally:
|
| 44 |
+
# Always clean up
|
| 45 |
+
__ENV_NAME__env.close()
|
| 46 |
+
```
|
| 47 |
+
|
| 48 |
+
That's it! The `__ENV_CLASS_NAME__Env.from_docker_image()` method handles:
|
| 49 |
+
- Starting the Docker container
|
| 50 |
+
- Waiting for the server to be ready
|
| 51 |
+
- Connecting to the environment
|
| 52 |
+
- Container cleanup when you call `close()`
|
| 53 |
+
|
| 54 |
+
## Building the Docker Image
|
| 55 |
+
|
| 56 |
+
Before using the environment, you need to build the Docker image:
|
| 57 |
+
|
| 58 |
+
```bash
|
| 59 |
+
# From project root
|
| 60 |
+
docker build -t __ENV_NAME__-env:latest -f server/Dockerfile .
|
| 61 |
+
```
|
| 62 |
+
|
| 63 |
+
## Deploying to Hugging Face Spaces
|
| 64 |
+
|
| 65 |
+
You can easily deploy your OpenEnv environment to Hugging Face Spaces using the `openenv push` command:
|
| 66 |
+
|
| 67 |
+
```bash
|
| 68 |
+
# From the environment directory (where openenv.yaml is located)
|
| 69 |
+
openenv push
|
| 70 |
+
|
| 71 |
+
# Or specify options
|
| 72 |
+
openenv push --namespace my-org --private
|
| 73 |
+
```
|
| 74 |
+
|
| 75 |
+
The `openenv push` command will:
|
| 76 |
+
1. Validate that the directory is an OpenEnv environment (checks for `openenv.yaml`)
|
| 77 |
+
2. Prepare a custom build for Hugging Face Docker space (enables web interface)
|
| 78 |
+
3. Upload to Hugging Face (ensuring you're logged in)
|
| 79 |
+
|
| 80 |
+
### Prerequisites
|
| 81 |
+
|
| 82 |
+
- Authenticate with Hugging Face: The command will prompt for login if not already authenticated
|
| 83 |
+
|
| 84 |
+
### Options
|
| 85 |
+
|
| 86 |
+
- `--directory`, `-d`: Directory containing the OpenEnv environment (defaults to current directory)
|
| 87 |
+
- `--repo-id`, `-r`: Repository ID in format 'username/repo-name' (defaults to 'username/env-name' from openenv.yaml)
|
| 88 |
+
- `--base-image`, `-b`: Base Docker image to use (overrides Dockerfile FROM)
|
| 89 |
+
- `--private`: Deploy the space as private (default: public)
|
| 90 |
+
|
| 91 |
+
### Examples
|
| 92 |
+
|
| 93 |
+
```bash
|
| 94 |
+
# Push to your personal namespace (defaults to username/env-name from openenv.yaml)
|
| 95 |
+
openenv push
|
| 96 |
+
|
| 97 |
+
# Push to a specific repository
|
| 98 |
+
openenv push --repo-id my-org/my-env
|
| 99 |
+
|
| 100 |
+
# Push with a custom base image
|
| 101 |
+
openenv push --base-image ghcr.io/meta-pytorch/openenv-base:latest
|
| 102 |
+
|
| 103 |
+
# Push as a private space
|
| 104 |
+
openenv push --private
|
| 105 |
+
|
| 106 |
+
# Combine options
|
| 107 |
+
openenv push --repo-id my-org/my-env --base-image custom-base:latest --private
|
| 108 |
+
```
|
| 109 |
+
|
| 110 |
+
After deployment, your space will be available at:
|
| 111 |
+
`https://huggingface.co/spaces/<repo-id>`
|
| 112 |
+
|
| 113 |
+
The deployed space includes:
|
| 114 |
+
- **Web Interface** at `/web` - Interactive UI for exploring the environment
|
| 115 |
+
- **API Documentation** at `/docs` - Full OpenAPI/Swagger interface
|
| 116 |
+
- **Health Check** at `/health` - Container health monitoring
|
| 117 |
+
- **WebSocket** at `/ws` - Persistent session endpoint for low-latency interactions
|
| 118 |
+
|
| 119 |
+
## Environment Details
|
| 120 |
+
|
| 121 |
+
### Action
|
| 122 |
+
**__ENV_CLASS_NAME__Action**: Contains a single field
|
| 123 |
+
- `message` (str) - The message to echo back
|
| 124 |
+
|
| 125 |
+
### Observation
|
| 126 |
+
**__ENV_CLASS_NAME__Observation**: Contains the echo response and metadata
|
| 127 |
+
- `echoed_message` (str) - The message echoed back
|
| 128 |
+
- `message_length` (int) - Length of the message
|
| 129 |
+
- `reward` (float) - Reward based on message length (length × 0.1)
|
| 130 |
+
- `done` (bool) - Always False for echo environment
|
| 131 |
+
- `metadata` (dict) - Additional info like step count
|
| 132 |
+
|
| 133 |
+
### Reward
|
| 134 |
+
The reward is calculated as: `message_length × 0.1`
|
| 135 |
+
- "Hi" → reward: 0.2
|
| 136 |
+
- "Hello, World!" → reward: 1.3
|
| 137 |
+
- Empty message → reward: 0.0
|
| 138 |
+
|
| 139 |
+
## Advanced Usage
|
| 140 |
+
|
| 141 |
+
### Connecting to an Existing Server
|
| 142 |
+
|
| 143 |
+
If you already have a __ENV_TITLE_NAME__ environment server running, you can connect directly:
|
| 144 |
+
|
| 145 |
+
```python
|
| 146 |
+
from __ENV_NAME__ import __ENV_CLASS_NAME__Env
|
| 147 |
+
|
| 148 |
+
# Connect to existing server
|
| 149 |
+
__ENV_NAME__env = __ENV_CLASS_NAME__Env(base_url="<ENV_HTTP_URL_HERE>")
|
| 150 |
+
|
| 151 |
+
# Use as normal
|
| 152 |
+
result = __ENV_NAME__env.reset()
|
| 153 |
+
result = __ENV_NAME__env.step(__ENV_CLASS_NAME__Action(message="Hello!"))
|
| 154 |
+
```
|
| 155 |
+
|
| 156 |
+
Note: When connecting to an existing server, `__ENV_NAME__env.close()` will NOT stop the server.
|
| 157 |
+
|
| 158 |
+
### Using the Context Manager
|
| 159 |
+
|
| 160 |
+
The client supports context manager usage for automatic connection management:
|
| 161 |
+
|
| 162 |
+
```python
|
| 163 |
+
from __ENV_NAME__ import __ENV_CLASS_NAME__Action, __ENV_CLASS_NAME__Env
|
| 164 |
+
|
| 165 |
+
# Connect with context manager (auto-connects and closes)
|
| 166 |
+
with __ENV_CLASS_NAME__Env(base_url="http://localhost:8000") as env:
|
| 167 |
+
result = env.reset()
|
| 168 |
+
print(f"Reset: {result.observation.echoed_message}")
|
| 169 |
+
# Multiple steps with low latency
|
| 170 |
+
for msg in ["Hello", "World", "!"]:
|
| 171 |
+
result = env.step(__ENV_CLASS_NAME__Action(message=msg))
|
| 172 |
+
print(f"Echoed: {result.observation.echoed_message}")
|
| 173 |
+
```
|
| 174 |
+
|
| 175 |
+
The client uses WebSocket connections for:
|
| 176 |
+
- **Lower latency**: No HTTP connection overhead per request
|
| 177 |
+
- **Persistent session**: Server maintains your environment state
|
| 178 |
+
- **Efficient for episodes**: Better for many sequential steps
|
| 179 |
+
|
| 180 |
+
### Concurrent WebSocket Sessions
|
| 181 |
+
|
| 182 |
+
The server supports multiple concurrent WebSocket connections. To enable this,
|
| 183 |
+
modify `server/app.py` to use factory mode:
|
| 184 |
+
|
| 185 |
+
```python
|
| 186 |
+
# In server/app.py - use factory mode for concurrent sessions
|
| 187 |
+
app = create_app(
|
| 188 |
+
__ENV_CLASS_NAME__Environment, # Pass class, not instance
|
| 189 |
+
__ENV_CLASS_NAME__Action,
|
| 190 |
+
__ENV_CLASS_NAME__Observation,
|
| 191 |
+
max_concurrent_envs=4, # Allow 4 concurrent sessions
|
| 192 |
+
)
|
| 193 |
+
```
|
| 194 |
+
|
| 195 |
+
Then multiple clients can connect simultaneously:
|
| 196 |
+
|
| 197 |
+
```python
|
| 198 |
+
from __ENV_NAME__ import __ENV_CLASS_NAME__Action, __ENV_CLASS_NAME__Env
|
| 199 |
+
from concurrent.futures import ThreadPoolExecutor
|
| 200 |
+
|
| 201 |
+
def run_episode(client_id: int):
|
| 202 |
+
with __ENV_CLASS_NAME__Env(base_url="http://localhost:8000") as env:
|
| 203 |
+
result = env.reset()
|
| 204 |
+
for i in range(10):
|
| 205 |
+
result = env.step(__ENV_CLASS_NAME__Action(message=f"Client {client_id}, step {i}"))
|
| 206 |
+
return client_id, result.observation.message_length
|
| 207 |
+
|
| 208 |
+
# Run 4 episodes concurrently
|
| 209 |
+
with ThreadPoolExecutor(max_workers=4) as executor:
|
| 210 |
+
results = list(executor.map(run_episode, range(4)))
|
| 211 |
+
```
|
| 212 |
+
|
| 213 |
+
## Development & Testing
|
| 214 |
+
|
| 215 |
+
### Direct Environment Testing
|
| 216 |
+
|
| 217 |
+
Test the environment logic directly without starting the HTTP server:
|
| 218 |
+
|
| 219 |
+
```bash
|
| 220 |
+
# From the server directory
|
| 221 |
+
python3 server/__ENV_NAME___environment.py
|
| 222 |
+
```
|
| 223 |
+
|
| 224 |
+
This verifies that:
|
| 225 |
+
- Environment resets correctly
|
| 226 |
+
- Step executes actions properly
|
| 227 |
+
- State tracking works
|
| 228 |
+
- Rewards are calculated correctly
|
| 229 |
+
|
| 230 |
+
### Running Locally
|
| 231 |
+
|
| 232 |
+
Run the server locally for development:
|
| 233 |
+
|
| 234 |
+
```bash
|
| 235 |
+
uvicorn server.app:app --reload
|
| 236 |
+
```
|
| 237 |
+
|
| 238 |
+
## Project Structure
|
| 239 |
+
|
| 240 |
+
```
|
| 241 |
+
__ENV_NAME__/
|
| 242 |
+
├── .dockerignore # Docker build exclusions
|
| 243 |
+
├── __init__.py # Module exports
|
| 244 |
+
├── README.md # This file
|
| 245 |
+
├── openenv.yaml # OpenEnv manifest
|
| 246 |
+
├── pyproject.toml # Project metadata and dependencies
|
| 247 |
+
├── uv.lock # Locked dependencies (generated)
|
| 248 |
+
├── client.py # __ENV_CLASS_NAME__Env client
|
| 249 |
+
├── models.py # Action and Observation models
|
| 250 |
+
└── server/
|
| 251 |
+
├── __init__.py # Server module exports
|
| 252 |
+
├── __ENV_NAME___environment.py # Core environment logic
|
| 253 |
+
├── app.py # FastAPI application (HTTP + WebSocket endpoints)
|
| 254 |
+
└── Dockerfile # Container image definition
|
| 255 |
+
```
|
src/openenv/cli/templates/openenv_env/__init__.py
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Meta Platforms, Inc. and affiliates.
|
| 2 |
+
# All rights reserved.
|
| 3 |
+
#
|
| 4 |
+
# This source code is licensed under the BSD-style license found in the
|
| 5 |
+
# LICENSE file in the root directory of this source tree.
|
| 6 |
+
|
| 7 |
+
"""__ENV_TITLE_NAME__ Environment."""
|
| 8 |
+
|
| 9 |
+
from .client import __ENV_CLASS_NAME__Env
|
| 10 |
+
from .models import __ENV_CLASS_NAME__Action, __ENV_CLASS_NAME__Observation
|
| 11 |
+
|
| 12 |
+
__all__ = [
|
| 13 |
+
"__ENV_CLASS_NAME__Action",
|
| 14 |
+
"__ENV_CLASS_NAME__Observation",
|
| 15 |
+
"__ENV_CLASS_NAME__Env",
|
| 16 |
+
]
|
src/openenv/cli/templates/openenv_env/client.py
ADDED
|
@@ -0,0 +1,99 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Meta Platforms, Inc. and affiliates.
|
| 2 |
+
# All rights reserved.
|
| 3 |
+
#
|
| 4 |
+
# This source code is licensed under the BSD-style license found in the
|
| 5 |
+
# LICENSE file in the root directory of this source tree.
|
| 6 |
+
|
| 7 |
+
"""__ENV_TITLE_NAME__ Environment Client."""
|
| 8 |
+
|
| 9 |
+
from typing import Dict
|
| 10 |
+
|
| 11 |
+
from openenv.core.client_types import StepResult
|
| 12 |
+
from openenv.core.env_server.types import State
|
| 13 |
+
from openenv.core import EnvClient
|
| 14 |
+
|
| 15 |
+
from .models import __ENV_CLASS_NAME__Action, __ENV_CLASS_NAME__Observation
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
class __ENV_CLASS_NAME__Env(
|
| 19 |
+
EnvClient[__ENV_CLASS_NAME__Action, __ENV_CLASS_NAME__Observation]
|
| 20 |
+
):
|
| 21 |
+
"""
|
| 22 |
+
Client for the __ENV_TITLE_NAME__ Environment.
|
| 23 |
+
|
| 24 |
+
This client maintains a persistent WebSocket connection to the environment server,
|
| 25 |
+
enabling efficient multi-step interactions with lower latency.
|
| 26 |
+
Each client instance has its own dedicated environment session on the server.
|
| 27 |
+
|
| 28 |
+
Example:
|
| 29 |
+
>>> # Connect to a running server
|
| 30 |
+
>>> with __ENV_CLASS_NAME__Env(base_url="http://localhost:8000") as client:
|
| 31 |
+
... result = client.reset()
|
| 32 |
+
... print(result.observation.echoed_message)
|
| 33 |
+
...
|
| 34 |
+
... result = client.step(__ENV_CLASS_NAME__Action(message="Hello!"))
|
| 35 |
+
... print(result.observation.echoed_message)
|
| 36 |
+
|
| 37 |
+
Example with Docker:
|
| 38 |
+
>>> # Automatically start container and connect
|
| 39 |
+
>>> client = __ENV_CLASS_NAME__Env.from_docker_image("__ENV_NAME__-env:latest")
|
| 40 |
+
>>> try:
|
| 41 |
+
... result = client.reset()
|
| 42 |
+
... result = client.step(__ENV_CLASS_NAME__Action(message="Test"))
|
| 43 |
+
... finally:
|
| 44 |
+
... client.close()
|
| 45 |
+
"""
|
| 46 |
+
|
| 47 |
+
def _step_payload(self, action: __ENV_CLASS_NAME__Action) -> Dict:
|
| 48 |
+
"""
|
| 49 |
+
Convert __ENV_CLASS_NAME__Action to JSON payload for step message.
|
| 50 |
+
|
| 51 |
+
Args:
|
| 52 |
+
action: __ENV_CLASS_NAME__Action instance
|
| 53 |
+
|
| 54 |
+
Returns:
|
| 55 |
+
Dictionary representation suitable for JSON encoding
|
| 56 |
+
"""
|
| 57 |
+
return {
|
| 58 |
+
"message": action.message,
|
| 59 |
+
}
|
| 60 |
+
|
| 61 |
+
def _parse_result(self, payload: Dict) -> StepResult[__ENV_CLASS_NAME__Observation]:
|
| 62 |
+
"""
|
| 63 |
+
Parse server response into StepResult[__ENV_CLASS_NAME__Observation].
|
| 64 |
+
|
| 65 |
+
Args:
|
| 66 |
+
payload: JSON response data from server
|
| 67 |
+
|
| 68 |
+
Returns:
|
| 69 |
+
StepResult with __ENV_CLASS_NAME__Observation
|
| 70 |
+
"""
|
| 71 |
+
obs_data = payload.get("observation", {})
|
| 72 |
+
observation = __ENV_CLASS_NAME__Observation(
|
| 73 |
+
echoed_message=obs_data.get("echoed_message", ""),
|
| 74 |
+
message_length=obs_data.get("message_length", 0),
|
| 75 |
+
done=payload.get("done", False),
|
| 76 |
+
reward=payload.get("reward"),
|
| 77 |
+
metadata=obs_data.get("metadata", {}),
|
| 78 |
+
)
|
| 79 |
+
|
| 80 |
+
return StepResult(
|
| 81 |
+
observation=observation,
|
| 82 |
+
reward=payload.get("reward"),
|
| 83 |
+
done=payload.get("done", False),
|
| 84 |
+
)
|
| 85 |
+
|
| 86 |
+
def _parse_state(self, payload: Dict) -> State:
|
| 87 |
+
"""
|
| 88 |
+
Parse server response into State object.
|
| 89 |
+
|
| 90 |
+
Args:
|
| 91 |
+
payload: JSON response from state request
|
| 92 |
+
|
| 93 |
+
Returns:
|
| 94 |
+
State object with episode_id and step_count
|
| 95 |
+
"""
|
| 96 |
+
return State(
|
| 97 |
+
episode_id=payload.get("episode_id"),
|
| 98 |
+
step_count=payload.get("step_count", 0),
|
| 99 |
+
)
|
src/openenv/cli/templates/openenv_env/models.py
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Meta Platforms, Inc. and affiliates.
|
| 2 |
+
# All rights reserved.
|
| 3 |
+
#
|
| 4 |
+
# This source code is licensed under the BSD-style license found in the
|
| 5 |
+
# LICENSE file in the root directory of this source tree.
|
| 6 |
+
|
| 7 |
+
"""
|
| 8 |
+
Data models for the __ENV_TITLE_NAME__ Environment.
|
| 9 |
+
|
| 10 |
+
The __ENV_NAME__ environment is a simple test environment that echoes back messages.
|
| 11 |
+
"""
|
| 12 |
+
|
| 13 |
+
from pydantic import Field
|
| 14 |
+
|
| 15 |
+
from openenv.core.env_server.types import Action, Observation
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
class __ENV_CLASS_NAME__Action(Action):
|
| 19 |
+
"""Action for the __ENV_TITLE_NAME__ environment - just a message to echo."""
|
| 20 |
+
|
| 21 |
+
message: str = Field(..., description="Message to echo back")
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
class __ENV_CLASS_NAME__Observation(Observation):
|
| 25 |
+
"""Observation from the __ENV_TITLE_NAME__ environment - the echoed message."""
|
| 26 |
+
|
| 27 |
+
echoed_message: str = Field(default="", description="The echoed message")
|
| 28 |
+
message_length: int = Field(default=0, description="Length of the echoed message")
|
src/openenv/cli/templates/openenv_env/openenv.yaml
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
spec_version: 1
|
| 2 |
+
name: __ENV_NAME__
|
| 3 |
+
type: space
|
| 4 |
+
runtime: fastapi
|
| 5 |
+
app: server.app:app
|
| 6 |
+
port: 8000
|
| 7 |
+
|