Upload folder using huggingface_hub
Browse files- .gitignore +0 -0
- .spacesignore +10 -0
- Dockerfile +71 -0
- README.md +99 -10
- __init__.py +16 -0
- assets/architecture.png +0 -0
- assets/output.png +0 -0
- client.py +99 -0
- graders.py +150 -0
- inference.py +228 -0
- models.py +148 -0
- openenv.yaml +35 -0
- output.json +6 -0
- pyproject.toml +45 -0
- server/__init__.py +7 -0
- server/app.py +276 -0
- server/my_env_environment.py +409 -0
- server/requirements.txt +6 -0
- submission_log.txt +0 -0
- test_env.py +24 -0
- test_local.py +20 -0
- validate_submission.py +132 -0
.gitignore
ADDED
|
Binary file (179 Bytes). View file
|
|
|
.spacesignore
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
__pycache__/
|
| 2 |
+
*.pyc
|
| 3 |
+
*.pyo
|
| 4 |
+
.git/
|
| 5 |
+
.vscode/
|
| 6 |
+
.idea/
|
| 7 |
+
*.log
|
| 8 |
+
test_*.py
|
| 9 |
+
validate_*.py
|
| 10 |
+
screenshots/
|
Dockerfile
ADDED
|
@@ -0,0 +1,71 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Meta Platforms, Inc. and affiliates.
|
| 2 |
+
# All rights reserved.
|
| 3 |
+
#
|
| 4 |
+
# This source code is licensed under the BSD-style license found in the
|
| 5 |
+
# LICENSE file in the root directory of this source tree.
|
| 6 |
+
|
| 7 |
+
# Multi-stage build using openenv-base
|
| 8 |
+
# This Dockerfile is flexible and works for both:
|
| 9 |
+
# - In-repo environments (with local OpenEnv sources)
|
| 10 |
+
# - Standalone environments (with openenv from PyPI/Git)
|
| 11 |
+
# The build script (openenv build) handles context detection and sets appropriate build args.
|
| 12 |
+
|
| 13 |
+
# Use OpenEnv base image
|
| 14 |
+
FROM ghcr.io/meta-pytorch/openenv-base:latest AS builder
|
| 15 |
+
|
| 16 |
+
WORKDIR /app
|
| 17 |
+
|
| 18 |
+
# Install system dependencies
|
| 19 |
+
RUN apt-get update && \
|
| 20 |
+
apt-get install -y --no-install-recommends \
|
| 21 |
+
curl \
|
| 22 |
+
ca-certificates && \
|
| 23 |
+
rm -rf /var/lib/apt/lists/*
|
| 24 |
+
|
| 25 |
+
# Copy entire environment
|
| 26 |
+
COPY . /app/env
|
| 27 |
+
|
| 28 |
+
WORKDIR /app/env
|
| 29 |
+
|
| 30 |
+
# Install uv if not present
|
| 31 |
+
RUN if ! command -v uv >/dev/null 2>&1; then \
|
| 32 |
+
curl -LsSf https://astral.sh/uv/install.sh | sh; \
|
| 33 |
+
fi
|
| 34 |
+
|
| 35 |
+
ENV PATH="/root/.cargo/bin:$PATH"
|
| 36 |
+
|
| 37 |
+
# Sync dependencies
|
| 38 |
+
RUN --mount=type=cache,target=/root/.cache/uv \
|
| 39 |
+
if [ -f uv.lock ]; then \
|
| 40 |
+
uv sync --frozen --no-dev; \
|
| 41 |
+
else \
|
| 42 |
+
uv sync --no-dev; \
|
| 43 |
+
fi
|
| 44 |
+
|
| 45 |
+
# Install openenv-core
|
| 46 |
+
RUN --mount=type=cache,target=/root/.cache/uv \
|
| 47 |
+
if [ -f uv.lock ]; then \
|
| 48 |
+
uv pip install openenv-core --python .venv/bin/python; \
|
| 49 |
+
else \
|
| 50 |
+
uv pip install openenv-core --python .venv/bin/python; \
|
| 51 |
+
fi
|
| 52 |
+
|
| 53 |
+
# Runtime stage
|
| 54 |
+
FROM ghcr.io/meta-pytorch/openenv-base:latest
|
| 55 |
+
|
| 56 |
+
WORKDIR /app
|
| 57 |
+
|
| 58 |
+
# Copy virtual environment and app
|
| 59 |
+
COPY --from=builder /app/env/.venv /app/.venv
|
| 60 |
+
COPY --from=builder /app/env /app/env
|
| 61 |
+
|
| 62 |
+
ENV PATH="/app/.venv/bin:$PATH"
|
| 63 |
+
ENV PYTHONPATH="/app/env:$PYTHONPATH"
|
| 64 |
+
|
| 65 |
+
WORKDIR /app/env
|
| 66 |
+
|
| 67 |
+
# Expose port
|
| 68 |
+
EXPOSE 7860
|
| 69 |
+
|
| 70 |
+
# Start the server
|
| 71 |
+
CMD ["uvicorn", "server.app:app", "--host", "0.0.0.0", "--port", "7860"]
|
README.md
CHANGED
|
@@ -1,10 +1,99 @@
|
|
| 1 |
-
---
|
| 2 |
-
title:
|
| 3 |
-
emoji:
|
| 4 |
-
colorFrom:
|
| 5 |
-
colorTo:
|
| 6 |
-
sdk: docker
|
| 7 |
-
pinned: false
|
| 8 |
-
---
|
| 9 |
-
|
| 10 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
---
|
| 2 |
+
title: OpenOps Incident Commander
|
| 3 |
+
emoji: ๐จ
|
| 4 |
+
colorFrom: blue
|
| 5 |
+
colorTo: purple
|
| 6 |
+
sdk: docker
|
| 7 |
+
pinned: false
|
| 8 |
+
---
|
| 9 |
+
# OpenOps: AI Incident Commander Environment
|
| 10 |
+
|
| 11 |
+
Production incident management environment where AI agents learn to handle real-world outages.
|
| 12 |
+
|
| 13 |
+
## Overview
|
| 14 |
+
|
| 15 |
+
OpenOps simulates production incidents requiring an AI Incident Commander to:
|
| 16 |
+
- Investigate alerts and logs
|
| 17 |
+
- Identify root causes
|
| 18 |
+
- Execute mitigation actions (restart/rollback/scale)
|
| 19 |
+
- Communicate with teams and users
|
| 20 |
+
- Resolve incidents quickly to minimize revenue loss
|
| 21 |
+
|
| 22 |
+
## Environment Specification
|
| 23 |
+
|
| 24 |
+
### Observation Space
|
| 25 |
+
```python
|
| 26 |
+
{
|
| 27 |
+
"active_alerts": List[str],
|
| 28 |
+
"service_status": Dict[str, str],
|
| 29 |
+
"recent_logs": Dict[str, List[str]],
|
| 30 |
+
"metrics_summary": Dict[str, float],
|
| 31 |
+
"customer_complaints": int,
|
| 32 |
+
"time_elapsed": int,
|
| 33 |
+
"revenue_loss": float,
|
| 34 |
+
"teams_notified": bool,
|
| 35 |
+
"status_page_updated": bool,
|
| 36 |
+
"user_communication_sent": bool
|
| 37 |
+
}
|
| 38 |
+
```
|
| 39 |
+
|
| 40 |
+
### Action Space (21 actions)
|
| 41 |
+
- 0: read_alerts
|
| 42 |
+
- 1-4: inspect_logs_{service}
|
| 43 |
+
- 5-8: check_metrics_{service}
|
| 44 |
+
- 9-12: restart_{service}
|
| 45 |
+
- 13-14: rollback_{service}
|
| 46 |
+
- 15-16: scale_{service}
|
| 47 |
+
- 17-19: Communication actions
|
| 48 |
+
- 20: resolve_incident
|
| 49 |
+
|
| 50 |
+
### Three Tasks
|
| 51 |
+
|
| 52 |
+
**Task 1 (Easy): Simple API Crash**
|
| 53 |
+
- API service down due to OOM
|
| 54 |
+
- Solution: Inspect logs โ Restart API
|
| 55 |
+
|
| 56 |
+
**Task 2 (Medium): Bad Deployment**
|
| 57 |
+
- Database deployment broke queries
|
| 58 |
+
- Solution: Inspect logs โ Rollback deployment โ Notify team
|
| 59 |
+
|
| 60 |
+
**Task 3 (Hard): Cascading Failure**
|
| 61 |
+
- Database overload โ API timeouts โ Customer impact
|
| 62 |
+
- Solution: Inspect both services โ Scale database โ Restart API โ Communicate
|
| 63 |
+
|
| 64 |
+
## Installation
|
| 65 |
+
```bash
|
| 66 |
+
pip install -r server/requirements.txt
|
| 67 |
+
```
|
| 68 |
+
|
| 69 |
+
## Usage
|
| 70 |
+
|
| 71 |
+
### Run Locally
|
| 72 |
+
```bash
|
| 73 |
+
cd server
|
| 74 |
+
uvicorn app:app --reload
|
| 75 |
+
```
|
| 76 |
+
|
| 77 |
+
### Run Inference
|
| 78 |
+
```bash
|
| 79 |
+
export OPENAI_API_KEY="your-key"
|
| 80 |
+
python inference.py
|
| 81 |
+
```
|
| 82 |
+
|
| 83 |
+
## Grading
|
| 84 |
+
|
| 85 |
+
Each task scored 0.0-1.0 based on:
|
| 86 |
+
- Investigation quality
|
| 87 |
+
- Correct mitigation actions
|
| 88 |
+
- Communication
|
| 89 |
+
- Successful resolution
|
| 90 |
+
|
| 91 |
+
## Deployment
|
| 92 |
+
|
| 93 |
+
Deploy to HuggingFace Spaces:
|
| 94 |
+
```bash
|
| 95 |
+
openenv push
|
| 96 |
+
```
|
| 97 |
+
|
| 98 |
+
## ๐ Sample Output
|
| 99 |
+

|
__init__.py
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Meta Platforms, Inc. and affiliates.
|
| 2 |
+
# All rights reserved.
|
| 3 |
+
#
|
| 4 |
+
# This source code is licensed under the BSD-style license found in the
|
| 5 |
+
# LICENSE file in the root directory of this source tree.
|
| 6 |
+
|
| 7 |
+
"""My Env Environment."""
|
| 8 |
+
|
| 9 |
+
from .client import MyEnv
|
| 10 |
+
from .models import MyAction, MyObservation
|
| 11 |
+
|
| 12 |
+
__all__ = [
|
| 13 |
+
"MyAction",
|
| 14 |
+
"MyObservation",
|
| 15 |
+
"MyEnv",
|
| 16 |
+
]
|
assets/architecture.png
ADDED
|
assets/output.png
ADDED
|
client.py
ADDED
|
@@ -0,0 +1,99 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Meta Platforms, Inc. and affiliates.
|
| 2 |
+
# All rights reserved.
|
| 3 |
+
#
|
| 4 |
+
# This source code is licensed under the BSD-style license found in the
|
| 5 |
+
# LICENSE file in the root directory of this source tree.
|
| 6 |
+
|
| 7 |
+
"""My Env Environment Client."""
|
| 8 |
+
|
| 9 |
+
from typing import Dict
|
| 10 |
+
|
| 11 |
+
from openenv.core import EnvClient
|
| 12 |
+
from openenv.core.client_types import StepResult
|
| 13 |
+
from openenv.core.env_server.types import State
|
| 14 |
+
|
| 15 |
+
from .models import MyAction, MyObservation
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
class MyEnv(
|
| 19 |
+
EnvClient[MyAction, MyObservation, State]
|
| 20 |
+
):
|
| 21 |
+
"""
|
| 22 |
+
Client for the My Env Environment.
|
| 23 |
+
|
| 24 |
+
This client maintains a persistent WebSocket connection to the environment server,
|
| 25 |
+
enabling efficient multi-step interactions with lower latency.
|
| 26 |
+
Each client instance has its own dedicated environment session on the server.
|
| 27 |
+
|
| 28 |
+
Example:
|
| 29 |
+
>>> # Connect to a running server
|
| 30 |
+
>>> with MyEnv(base_url="http://localhost:8000") as client:
|
| 31 |
+
... result = client.reset()
|
| 32 |
+
... print(result.observation.echoed_message)
|
| 33 |
+
...
|
| 34 |
+
... result = client.step(MyAction(message="Hello!"))
|
| 35 |
+
... print(result.observation.echoed_message)
|
| 36 |
+
|
| 37 |
+
Example with Docker:
|
| 38 |
+
>>> # Automatically start container and connect
|
| 39 |
+
>>> client = MyEnv.from_docker_image("my_env-env:latest")
|
| 40 |
+
>>> try:
|
| 41 |
+
... result = client.reset()
|
| 42 |
+
... result = client.step(MyAction(message="Test"))
|
| 43 |
+
... finally:
|
| 44 |
+
... client.close()
|
| 45 |
+
"""
|
| 46 |
+
|
| 47 |
+
def _step_payload(self, action: MyAction) -> Dict:
|
| 48 |
+
"""
|
| 49 |
+
Convert MyAction to JSON payload for step message.
|
| 50 |
+
|
| 51 |
+
Args:
|
| 52 |
+
action: MyAction instance
|
| 53 |
+
|
| 54 |
+
Returns:
|
| 55 |
+
Dictionary representation suitable for JSON encoding
|
| 56 |
+
"""
|
| 57 |
+
return {
|
| 58 |
+
"message": action.message,
|
| 59 |
+
}
|
| 60 |
+
|
| 61 |
+
def _parse_result(self, payload: Dict) -> StepResult[MyObservation]:
|
| 62 |
+
"""
|
| 63 |
+
Parse server response into StepResult[MyObservation].
|
| 64 |
+
|
| 65 |
+
Args:
|
| 66 |
+
payload: JSON response data from server
|
| 67 |
+
|
| 68 |
+
Returns:
|
| 69 |
+
StepResult with MyObservation
|
| 70 |
+
"""
|
| 71 |
+
obs_data = payload.get("observation", {})
|
| 72 |
+
observation = MyObservation(
|
| 73 |
+
echoed_message=obs_data.get("echoed_message", ""),
|
| 74 |
+
message_length=obs_data.get("message_length", 0),
|
| 75 |
+
done=payload.get("done", False),
|
| 76 |
+
reward=payload.get("reward"),
|
| 77 |
+
metadata=obs_data.get("metadata", {}),
|
| 78 |
+
)
|
| 79 |
+
|
| 80 |
+
return StepResult(
|
| 81 |
+
observation=observation,
|
| 82 |
+
reward=payload.get("reward"),
|
| 83 |
+
done=payload.get("done", False),
|
| 84 |
+
)
|
| 85 |
+
|
| 86 |
+
def _parse_state(self, payload: Dict) -> State:
|
| 87 |
+
"""
|
| 88 |
+
Parse server response into State object.
|
| 89 |
+
|
| 90 |
+
Args:
|
| 91 |
+
payload: JSON response from state request
|
| 92 |
+
|
| 93 |
+
Returns:
|
| 94 |
+
State object with episode_id and step_count
|
| 95 |
+
"""
|
| 96 |
+
return State(
|
| 97 |
+
episode_id=payload.get("episode_id"),
|
| 98 |
+
step_count=payload.get("step_count", 0),
|
| 99 |
+
)
|
graders.py
ADDED
|
@@ -0,0 +1,150 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Grading functions for OpenOps tasks
|
| 3 |
+
Each grader returns a score between 0.0 and 1.0
|
| 4 |
+
"""
|
| 5 |
+
|
| 6 |
+
from typing import Callable
|
| 7 |
+
from server.my_env_environment import MyEnvEnvironment
|
| 8 |
+
|
| 9 |
+
|
| 10 |
+
def grade_task_1(env: MyEnvEnvironment) -> float:
|
| 11 |
+
"""
|
| 12 |
+
Grade Task 1: Simple API Crash
|
| 13 |
+
|
| 14 |
+
Scoring criteria:
|
| 15 |
+
- Investigation (30%): Read alerts + inspected API logs
|
| 16 |
+
- Mitigation (50%): Restarted API service
|
| 17 |
+
- Resolution (20%): Incident resolved
|
| 18 |
+
|
| 19 |
+
Args:
|
| 20 |
+
env: Environment instance after task completion
|
| 21 |
+
|
| 22 |
+
Returns:
|
| 23 |
+
Score between 0.0 and 1.0
|
| 24 |
+
"""
|
| 25 |
+
score = 0.0
|
| 26 |
+
|
| 27 |
+
# Investigation (30%)
|
| 28 |
+
if env.alerts_read:
|
| 29 |
+
score += 0.15
|
| 30 |
+
if "api" in env.logs_inspected:
|
| 31 |
+
score += 0.15
|
| 32 |
+
|
| 33 |
+
# Mitigation (50%)
|
| 34 |
+
if "api" in env.services_restarted:
|
| 35 |
+
score += 0.50
|
| 36 |
+
|
| 37 |
+
# Resolution (20%)
|
| 38 |
+
if env.incident_resolved:
|
| 39 |
+
score += 0.20
|
| 40 |
+
|
| 41 |
+
return min(1.0, score)
|
| 42 |
+
|
| 43 |
+
|
| 44 |
+
def grade_task_2(env: MyEnvEnvironment) -> float:
|
| 45 |
+
"""
|
| 46 |
+
Grade Task 2: Bad Deployment
|
| 47 |
+
|
| 48 |
+
Scoring criteria:
|
| 49 |
+
- Investigation (25%): Read alerts + inspected database logs
|
| 50 |
+
- Mitigation (45%): Rolled back database
|
| 51 |
+
- Communication (15%): Notified team
|
| 52 |
+
- Resolution (15%): Incident resolved
|
| 53 |
+
|
| 54 |
+
Args:
|
| 55 |
+
env: Environment instance after task completion
|
| 56 |
+
|
| 57 |
+
Returns:
|
| 58 |
+
Score between 0.0 and 1.0
|
| 59 |
+
"""
|
| 60 |
+
score = 0.0
|
| 61 |
+
|
| 62 |
+
# Investigation (25%)
|
| 63 |
+
if env.alerts_read:
|
| 64 |
+
score += 0.10
|
| 65 |
+
if "database" in env.logs_inspected:
|
| 66 |
+
score += 0.15
|
| 67 |
+
|
| 68 |
+
# Mitigation (45%)
|
| 69 |
+
if "database" in env.services_rolled_back:
|
| 70 |
+
score += 0.45
|
| 71 |
+
|
| 72 |
+
# Communication (15%)
|
| 73 |
+
if env.teams_notified:
|
| 74 |
+
score += 0.15
|
| 75 |
+
|
| 76 |
+
# Resolution (15%)
|
| 77 |
+
if env.incident_resolved:
|
| 78 |
+
score += 0.15
|
| 79 |
+
|
| 80 |
+
return min(1.0, score)
|
| 81 |
+
|
| 82 |
+
|
| 83 |
+
def grade_task_3(env: MyEnvEnvironment) -> float:
|
| 84 |
+
"""
|
| 85 |
+
Grade Task 3: Cascading Failure
|
| 86 |
+
|
| 87 |
+
Scoring criteria:
|
| 88 |
+
- Investigation (20%): Read alerts + inspected both services
|
| 89 |
+
- Mitigation (50%): Scaled database + restarted API
|
| 90 |
+
- Communication (15%): Notified team + updated status
|
| 91 |
+
- Resolution (15%): Incident resolved
|
| 92 |
+
|
| 93 |
+
Args:
|
| 94 |
+
env: Environment instance after task completion
|
| 95 |
+
|
| 96 |
+
Returns:
|
| 97 |
+
Score between 0.0 and 1.0
|
| 98 |
+
"""
|
| 99 |
+
score = 0.0
|
| 100 |
+
|
| 101 |
+
# Investigation (20%)
|
| 102 |
+
if env.alerts_read:
|
| 103 |
+
score += 0.05
|
| 104 |
+
if "database" in env.logs_inspected:
|
| 105 |
+
score += 0.075
|
| 106 |
+
if "api" in env.logs_inspected:
|
| 107 |
+
score += 0.075
|
| 108 |
+
|
| 109 |
+
# Mitigation (50%)
|
| 110 |
+
if "database" in env.services_scaled:
|
| 111 |
+
score += 0.25
|
| 112 |
+
if "api" in env.services_restarted:
|
| 113 |
+
score += 0.25
|
| 114 |
+
|
| 115 |
+
# Communication (15%)
|
| 116 |
+
if env.teams_notified:
|
| 117 |
+
score += 0.075
|
| 118 |
+
if env.status_page_updated:
|
| 119 |
+
score += 0.075
|
| 120 |
+
|
| 121 |
+
# Resolution (15%)
|
| 122 |
+
if env.incident_resolved:
|
| 123 |
+
score += 0.15
|
| 124 |
+
|
| 125 |
+
return min(1.0, score)
|
| 126 |
+
|
| 127 |
+
|
| 128 |
+
def get_grader(task_id: int) -> Callable[[MyEnvEnvironment], float]:
|
| 129 |
+
"""
|
| 130 |
+
Get the appropriate grader function for a task.
|
| 131 |
+
|
| 132 |
+
Args:
|
| 133 |
+
task_id: Task ID (1, 2, or 3)
|
| 134 |
+
|
| 135 |
+
Returns:
|
| 136 |
+
Grader function
|
| 137 |
+
|
| 138 |
+
Raises:
|
| 139 |
+
ValueError: If task_id is invalid
|
| 140 |
+
"""
|
| 141 |
+
graders = {
|
| 142 |
+
1: grade_task_1,
|
| 143 |
+
2: grade_task_2,
|
| 144 |
+
3: grade_task_3
|
| 145 |
+
}
|
| 146 |
+
|
| 147 |
+
if task_id not in graders:
|
| 148 |
+
raise ValueError(f"Invalid task_id: {task_id}. Must be 1, 2, or 3.")
|
| 149 |
+
|
| 150 |
+
return graders[task_id]
|
inference.py
ADDED
|
@@ -0,0 +1,228 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
OpenOps FINAL Agent - Optimized Playbooks with Required Logging
|
| 3 |
+
This agent implements optimized playbooks for each task, with smart incident type detection.
|
| 4 |
+
It includes the required logging for start, each step, and end of the episode.
|
| 5 |
+
"""
|
| 6 |
+
|
| 7 |
+
import os
|
| 8 |
+
import json
|
| 9 |
+
import sys
|
| 10 |
+
from openai import OpenAI
|
| 11 |
+
from models import IncidentAction
|
| 12 |
+
from server.my_env_environment import MyEnvEnvironment
|
| 13 |
+
from graders import get_grader
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
# =========================================================
|
| 17 |
+
# ENV VARIABLES
|
| 18 |
+
# =========================================================
|
| 19 |
+
API_BASE_URL = os.getenv("API_BASE_URL", "https://api.groq.com/openai/v1")
|
| 20 |
+
MODEL_NAME = os.getenv("MODEL_NAME", "llama-3.3-70b-versatile")
|
| 21 |
+
API_KEY = os.getenv("HF_TOKEN") or os.getenv("OPENAI_API_KEY") or os.getenv("GROQ_API_KEY")
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
# =========================================================
|
| 25 |
+
# REQUIRED LOGGING
|
| 26 |
+
# =========================================================
|
| 27 |
+
def log_start(task_id: int):
|
| 28 |
+
"""Hackathon-required start log."""
|
| 29 |
+
print(f"[START] task_id={task_id}")
|
| 30 |
+
sys.stdout.flush()
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
def log_step(step_num: int, action_id: int, action_name: str, reward: float):
|
| 34 |
+
"""Hackathon-required step log."""
|
| 35 |
+
log_data = {
|
| 36 |
+
"step": step_num,
|
| 37 |
+
"action_id": action_id,
|
| 38 |
+
"action_name": action_name,
|
| 39 |
+
"reward": round(reward, 4)
|
| 40 |
+
}
|
| 41 |
+
print(f"[STEP] {json.dumps(log_data)}")
|
| 42 |
+
sys.stdout.flush()
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
def log_end(task_id: int, total_reward: float, final_score: float, resolved: bool):
|
| 46 |
+
"""Hackathon-required end log."""
|
| 47 |
+
log_data = {
|
| 48 |
+
"task_id": task_id,
|
| 49 |
+
"total_reward": round(total_reward, 4),
|
| 50 |
+
"final_score": round(final_score, 4),
|
| 51 |
+
"incident_resolved": resolved
|
| 52 |
+
}
|
| 53 |
+
print(f"[END] {json.dumps(log_data)}")
|
| 54 |
+
sys.stdout.flush()
|
| 55 |
+
|
| 56 |
+
|
| 57 |
+
# =========================================================
|
| 58 |
+
# INCIDENT DETECTION
|
| 59 |
+
# =========================================================
|
| 60 |
+
def detect_incident_type(observation) -> str:
|
| 61 |
+
"""Smart detection based on alerts, logs, and service status."""
|
| 62 |
+
text = (
|
| 63 |
+
str(observation.active_alerts) +
|
| 64 |
+
str(observation.recent_logs) +
|
| 65 |
+
str(observation.service_status)
|
| 66 |
+
).lower()
|
| 67 |
+
|
| 68 |
+
# Task 2/3: Database-related incidents
|
| 69 |
+
if any(word in text for word in [
|
| 70 |
+
"database", "db", "sql", "connection pool",
|
| 71 |
+
"too many connections", "timeout connecting",
|
| 72 |
+
"connection refused", "postgres", "mysql",
|
| 73 |
+
"pool exhausted", "lock wait", "slow query"
|
| 74 |
+
]):
|
| 75 |
+
return "database"
|
| 76 |
+
|
| 77 |
+
# Task 3: Memory incidents
|
| 78 |
+
if any(word in text for word in [
|
| 79 |
+
"memory", "oom", "out of memory",
|
| 80 |
+
"killed process", "high memory"
|
| 81 |
+
]):
|
| 82 |
+
return "memory"
|
| 83 |
+
|
| 84 |
+
# Task 1: Default to API
|
| 85 |
+
return "api"
|
| 86 |
+
|
| 87 |
+
|
| 88 |
+
# =========================================================
|
| 89 |
+
# OPTIMIZED PLAYBOOKS
|
| 90 |
+
# =========================================================
|
| 91 |
+
PLAYBOOKS = {
|
| 92 |
+
# Task 1: API crash
|
| 93 |
+
"api": [
|
| 94 |
+
0, # read_alerts
|
| 95 |
+
1, # inspect_logs_api
|
| 96 |
+
9, # restart_api
|
| 97 |
+
20 # resolve
|
| 98 |
+
],
|
| 99 |
+
|
| 100 |
+
# Task 2 & partial Task 3: Database issues
|
| 101 |
+
"database": [
|
| 102 |
+
0, # read_alerts
|
| 103 |
+
2, # inspect_logs_database
|
| 104 |
+
14, # rollback_database (works for Task 2)
|
| 105 |
+
16, # scale_database (works for Task 3)
|
| 106 |
+
1, # inspect_logs_api
|
| 107 |
+
9, # restart_api
|
| 108 |
+
17, # notify_team
|
| 109 |
+
18, # update_status_page
|
| 110 |
+
20 # resolve
|
| 111 |
+
],
|
| 112 |
+
|
| 113 |
+
# Task 3 alternate: Memory leak
|
| 114 |
+
"memory": [
|
| 115 |
+
0, # read_alerts
|
| 116 |
+
1, # inspect_logs_api
|
| 117 |
+
15, # scale_api
|
| 118 |
+
9, # restart_api
|
| 119 |
+
17, # notify_team
|
| 120 |
+
18, # update_status_page
|
| 121 |
+
20 # resolve
|
| 122 |
+
]
|
| 123 |
+
}
|
| 124 |
+
|
| 125 |
+
|
| 126 |
+
# =========================================================
|
| 127 |
+
# RUN SINGLE TASK
|
| 128 |
+
# =========================================================
|
| 129 |
+
def run_task(task_id: int, max_steps: int = 30) -> dict:
|
| 130 |
+
"""
|
| 131 |
+
Execute task with smart detection + required logging.
|
| 132 |
+
|
| 133 |
+
Args:
|
| 134 |
+
task_id: 1 (easy), 2 (medium), or 3 (hard)
|
| 135 |
+
max_steps: Maximum steps allowed
|
| 136 |
+
|
| 137 |
+
Returns:
|
| 138 |
+
Task results
|
| 139 |
+
"""
|
| 140 |
+
# REQUIRED: Log start
|
| 141 |
+
log_start(task_id)
|
| 142 |
+
|
| 143 |
+
# Initialize environment
|
| 144 |
+
env = MyEnvEnvironment()
|
| 145 |
+
obs = env.reset(task_id=task_id)
|
| 146 |
+
|
| 147 |
+
# Detect incident type
|
| 148 |
+
incident_type = detect_incident_type(obs)
|
| 149 |
+
|
| 150 |
+
# Get optimal playbook
|
| 151 |
+
playbook = PLAYBOOKS.get(incident_type, PLAYBOOKS["api"])
|
| 152 |
+
|
| 153 |
+
# Execute playbook with logging
|
| 154 |
+
step_num = 0
|
| 155 |
+
done = False
|
| 156 |
+
|
| 157 |
+
for action_id in playbook:
|
| 158 |
+
if done or step_num >= max_steps:
|
| 159 |
+
break
|
| 160 |
+
|
| 161 |
+
step_num += 1
|
| 162 |
+
action_name = env.ACTION_NAMES.get(action_id, "unknown")
|
| 163 |
+
action = IncidentAction(action_id=action_id, task_id=task_id)
|
| 164 |
+
|
| 165 |
+
obs = env.step(action)
|
| 166 |
+
|
| 167 |
+
# REQUIRED: Log each step
|
| 168 |
+
log_step(step_num, action_id, action_name, obs.reward)
|
| 169 |
+
|
| 170 |
+
done = obs.done
|
| 171 |
+
|
| 172 |
+
# Calculate final score
|
| 173 |
+
grader = get_grader(task_id)
|
| 174 |
+
final_score = grader(env)
|
| 175 |
+
|
| 176 |
+
# REQUIRED: Log end
|
| 177 |
+
log_end(task_id, env.total_reward, final_score, env.incident_resolved)
|
| 178 |
+
|
| 179 |
+
return {
|
| 180 |
+
"task_id": task_id,
|
| 181 |
+
"total_reward": env.total_reward,
|
| 182 |
+
"final_score": final_score,
|
| 183 |
+
"incident_resolved": env.incident_resolved,
|
| 184 |
+
"steps_taken": step_num
|
| 185 |
+
}
|
| 186 |
+
|
| 187 |
+
|
| 188 |
+
# =========================================================
|
| 189 |
+
# MAIN EVALUATION
|
| 190 |
+
# =========================================================
|
| 191 |
+
def main():
|
| 192 |
+
"""Run all three tasks."""
|
| 193 |
+
print("="*60)
|
| 194 |
+
print("OpenOps: Optimized Playbook Agent")
|
| 195 |
+
print("="*60)
|
| 196 |
+
print()
|
| 197 |
+
|
| 198 |
+
results = []
|
| 199 |
+
|
| 200 |
+
for task_id in [1, 2, 3]:
|
| 201 |
+
try:
|
| 202 |
+
result = run_task(task_id)
|
| 203 |
+
results.append(result)
|
| 204 |
+
except Exception as e:
|
| 205 |
+
print(f"[ERROR] Task {task_id}: {e}", file=sys.stderr)
|
| 206 |
+
results.append({
|
| 207 |
+
"task_id": task_id,
|
| 208 |
+
"total_reward": 0.0,
|
| 209 |
+
"final_score": 0.0,
|
| 210 |
+
"incident_resolved": False,
|
| 211 |
+
"steps_taken": 0
|
| 212 |
+
})
|
| 213 |
+
|
| 214 |
+
# Summary
|
| 215 |
+
print()
|
| 216 |
+
print("="*60)
|
| 217 |
+
print("SUMMARY")
|
| 218 |
+
print("="*60)
|
| 219 |
+
for r in results:
|
| 220 |
+
print(f"Task {r['task_id']}: Score={r['final_score']:.2f}, Resolved={r['incident_resolved']}")
|
| 221 |
+
|
| 222 |
+
avg_score = sum(r['final_score'] for r in results) / len(results)
|
| 223 |
+
print(f"\nAverage Score: {avg_score:.2f}")
|
| 224 |
+
print("="*60)
|
| 225 |
+
|
| 226 |
+
|
| 227 |
+
if __name__ == "__main__":
|
| 228 |
+
main()
|
models.py
ADDED
|
@@ -0,0 +1,148 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Meta Platforms, Inc.
|
| 2 |
+
"""
|
| 3 |
+
Pydantic models for OpenOps environment
|
| 4 |
+
"""
|
| 5 |
+
|
| 6 |
+
from typing import Dict, List, Optional
|
| 7 |
+
from pydantic import BaseModel, Field
|
| 8 |
+
|
| 9 |
+
|
| 10 |
+
class IncidentAction(BaseModel):
|
| 11 |
+
"""
|
| 12 |
+
Action taken by the agent.
|
| 13 |
+
|
| 14 |
+
Represents a single action in the incident management workflow.
|
| 15 |
+
"""
|
| 16 |
+
action_id: int = Field(..., ge=0, le=20, description="Action ID (0-20)")
|
| 17 |
+
task_id: int = Field(default=1, ge=1, le=3, description="Task ID (1=easy, 2=medium, 3=hard)")
|
| 18 |
+
|
| 19 |
+
class Config:
|
| 20 |
+
json_schema_extra = {
|
| 21 |
+
"example": {
|
| 22 |
+
"action_id": 0,
|
| 23 |
+
"task_id": 1
|
| 24 |
+
}
|
| 25 |
+
}
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
class IncidentObservation(BaseModel):
|
| 29 |
+
"""
|
| 30 |
+
Observation returned to agent after each step.
|
| 31 |
+
|
| 32 |
+
Contains partial information about the system state (investigation reveals more).
|
| 33 |
+
"""
|
| 34 |
+
active_alerts: List[str] = Field(
|
| 35 |
+
default_factory=list,
|
| 36 |
+
description="List of active system alerts"
|
| 37 |
+
)
|
| 38 |
+
service_status: Dict[str, str] = Field(
|
| 39 |
+
default_factory=dict,
|
| 40 |
+
description="Status of each service (healthy/degraded/down)"
|
| 41 |
+
)
|
| 42 |
+
recent_logs: Dict[str, List[str]] = Field(
|
| 43 |
+
default_factory=dict,
|
| 44 |
+
description="Logs from inspected services only"
|
| 45 |
+
)
|
| 46 |
+
metrics_summary: Dict[str, Dict[str, float]] = Field(
|
| 47 |
+
default_factory=dict,
|
| 48 |
+
description="Metrics for checked services (CPU, memory, latency)"
|
| 49 |
+
)
|
| 50 |
+
customer_complaints: int = Field(
|
| 51 |
+
default=0,
|
| 52 |
+
description="Number of customer complaints received"
|
| 53 |
+
)
|
| 54 |
+
time_elapsed: int = Field(
|
| 55 |
+
default=0,
|
| 56 |
+
description="Minutes since incident started"
|
| 57 |
+
)
|
| 58 |
+
revenue_loss: float = Field(
|
| 59 |
+
default=0.0,
|
| 60 |
+
description="Estimated revenue loss in USD"
|
| 61 |
+
)
|
| 62 |
+
teams_notified: bool = Field(
|
| 63 |
+
default=False,
|
| 64 |
+
description="Whether engineering team has been notified"
|
| 65 |
+
)
|
| 66 |
+
status_page_updated: bool = Field(
|
| 67 |
+
default=False,
|
| 68 |
+
description="Whether public status page has been updated"
|
| 69 |
+
)
|
| 70 |
+
reward: float = Field(
|
| 71 |
+
default=0.0,
|
| 72 |
+
description="Reward received for this step"
|
| 73 |
+
)
|
| 74 |
+
done: bool = Field(
|
| 75 |
+
default=False,
|
| 76 |
+
description="Whether episode is complete"
|
| 77 |
+
)
|
| 78 |
+
|
| 79 |
+
class Config:
|
| 80 |
+
json_schema_extra = {
|
| 81 |
+
"example": {
|
| 82 |
+
"active_alerts": ["CRITICAL: API service down"],
|
| 83 |
+
"service_status": {
|
| 84 |
+
"api": "down",
|
| 85 |
+
"database": "healthy"
|
| 86 |
+
},
|
| 87 |
+
"recent_logs": {
|
| 88 |
+
"api": ["ERROR: Out of memory"]
|
| 89 |
+
},
|
| 90 |
+
"customer_complaints": 45,
|
| 91 |
+
"time_elapsed": 5,
|
| 92 |
+
"revenue_loss": 5000.0,
|
| 93 |
+
"teams_notified": False,
|
| 94 |
+
"status_page_updated": False,
|
| 95 |
+
"reward": 0.05,
|
| 96 |
+
"done": False
|
| 97 |
+
}
|
| 98 |
+
}
|
| 99 |
+
|
| 100 |
+
|
| 101 |
+
class IncidentState(BaseModel):
|
| 102 |
+
"""
|
| 103 |
+
Internal environment state (hidden from agent).
|
| 104 |
+
|
| 105 |
+
Contains ground truth about the incident for evaluation.
|
| 106 |
+
"""
|
| 107 |
+
task_id: int = Field(..., ge=1, le=3, description="Task difficulty level")
|
| 108 |
+
incident_type: str = Field(..., description="Type of incident")
|
| 109 |
+
affected_services: List[str] = Field(
|
| 110 |
+
default_factory=list,
|
| 111 |
+
description="Services affected by the incident"
|
| 112 |
+
)
|
| 113 |
+
root_cause: str = Field(..., description="Root cause of the incident")
|
| 114 |
+
service_status: Dict[str, str] = Field(
|
| 115 |
+
default_factory=dict,
|
| 116 |
+
description="Current status of all services"
|
| 117 |
+
)
|
| 118 |
+
correct_mitigation: List[str] = Field(
|
| 119 |
+
default_factory=list,
|
| 120 |
+
description="Correct mitigation actions for this incident"
|
| 121 |
+
)
|
| 122 |
+
revenue_loss: float = Field(
|
| 123 |
+
default=0.0,
|
| 124 |
+
description="Accumulated revenue loss"
|
| 125 |
+
)
|
| 126 |
+
customer_complaints: int = Field(
|
| 127 |
+
default=0,
|
| 128 |
+
description="Accumulated customer complaints"
|
| 129 |
+
)
|
| 130 |
+
|
| 131 |
+
class Config:
|
| 132 |
+
json_schema_extra = {
|
| 133 |
+
"example": {
|
| 134 |
+
"task_id": 1,
|
| 135 |
+
"incident_type": "api_crash",
|
| 136 |
+
"affected_services": ["api"],
|
| 137 |
+
"root_cause": "out_of_memory",
|
| 138 |
+
"service_status": {
|
| 139 |
+
"api": "down",
|
| 140 |
+
"database": "healthy",
|
| 141 |
+
"auth": "healthy",
|
| 142 |
+
"frontend": "degraded"
|
| 143 |
+
},
|
| 144 |
+
"correct_mitigation": ["restart_api"],
|
| 145 |
+
"revenue_loss": 0.0,
|
| 146 |
+
"customer_complaints": 0
|
| 147 |
+
}
|
| 148 |
+
}
|
openenv.yaml
ADDED
|
@@ -0,0 +1,35 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
name: openops
|
| 2 |
+
version: 0.1.0
|
| 3 |
+
description: "Production incident management environment for AI agents"
|
| 4 |
+
author: "Arya Singh"
|
| 5 |
+
tags:
|
| 6 |
+
- incident-response
|
| 7 |
+
- devops
|
| 8 |
+
- production-management
|
| 9 |
+
- real-world
|
| 10 |
+
|
| 11 |
+
environment_class: server.my_env_environment.MyEnvEnvironment
|
| 12 |
+
action_class: models.IncidentAction
|
| 13 |
+
observation_class: models.IncidentObservation
|
| 14 |
+
state_class: models.IncidentState
|
| 15 |
+
|
| 16 |
+
tasks:
|
| 17 |
+
- id: 1
|
| 18 |
+
name: "Simple API Crash"
|
| 19 |
+
difficulty: easy
|
| 20 |
+
description: "API service crashed - restart to restore"
|
| 21 |
+
|
| 22 |
+
- id: 2
|
| 23 |
+
name: "Bad Deployment"
|
| 24 |
+
difficulty: medium
|
| 25 |
+
description: "Database deployment broke queries - rollback needed"
|
| 26 |
+
|
| 27 |
+
- id: 3
|
| 28 |
+
name: "Cascading Failure"
|
| 29 |
+
difficulty: hard
|
| 30 |
+
description: "Database overload causing API failures - scale and restart"
|
| 31 |
+
|
| 32 |
+
grading:
|
| 33 |
+
task_1: graders.grade_task_1
|
| 34 |
+
task_2: graders.grade_task_2
|
| 35 |
+
task_3: graders.grade_task_3
|
output.json
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"service": "api",
|
| 3 |
+
"root_cause": "timeout",
|
| 4 |
+
"severity": "high",
|
| 5 |
+
"explanation": "Upstream service is slow or not responding in time."
|
| 6 |
+
}
|
pyproject.toml
ADDED
|
@@ -0,0 +1,45 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Meta Platforms, Inc. and affiliates.
|
| 2 |
+
# All rights reserved.
|
| 3 |
+
#
|
| 4 |
+
# This source code is licensed under the BSD-style license found in the
|
| 5 |
+
# LICENSE file in the root directory of this source tree.
|
| 6 |
+
|
| 7 |
+
[build-system]
|
| 8 |
+
requires = ["setuptools>=45", "wheel"]
|
| 9 |
+
build-backend = "setuptools.build_meta"
|
| 10 |
+
|
| 11 |
+
[project]
|
| 12 |
+
name = "openenv-my_env"
|
| 13 |
+
version = "0.1.0"
|
| 14 |
+
description = "My Env environment for OpenEnv"
|
| 15 |
+
requires-python = ">=3.10"
|
| 16 |
+
dependencies = [
|
| 17 |
+
# Core OpenEnv runtime (provides FastAPI server + HTTP client types)
|
| 18 |
+
# install from github
|
| 19 |
+
# "openenv-core[core] @ git+https://github.com/meta-pytorch/OpenEnv.git",
|
| 20 |
+
"openenv-core[core]>=0.2.2",
|
| 21 |
+
# Environment-specific dependencies
|
| 22 |
+
# Add all dependencies needed for your environment here
|
| 23 |
+
# Examples:
|
| 24 |
+
# "numpy>=1.19.0",
|
| 25 |
+
# "torch>=2.0.0",
|
| 26 |
+
# "gymnasium>=0.29.0",
|
| 27 |
+
# "openspiel>=1.0.0",
|
| 28 |
+
# "smolagents>=1.22.0,<2",
|
| 29 |
+
]
|
| 30 |
+
|
| 31 |
+
[project.optional-dependencies]
|
| 32 |
+
dev = [
|
| 33 |
+
"pytest>=8.0.0",
|
| 34 |
+
"pytest-cov>=4.0.0",
|
| 35 |
+
]
|
| 36 |
+
|
| 37 |
+
[project.scripts]
|
| 38 |
+
# Server entry point - enables running via: uv run --project . server
|
| 39 |
+
# or: python -m my_env.server.app
|
| 40 |
+
server = "my_env.server.app:main"
|
| 41 |
+
|
| 42 |
+
[tool.setuptools]
|
| 43 |
+
include-package-data = true
|
| 44 |
+
packages = ["my_env", "my_env.server"]
|
| 45 |
+
package-dir = { "my_env" = ".", "my_env.server" = "server" }
|
server/__init__.py
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
OpenOps Server Package
|
| 3 |
+
"""
|
| 4 |
+
|
| 5 |
+
from server.my_env_environment import MyEnvEnvironment
|
| 6 |
+
|
| 7 |
+
__all__ = ["MyEnvEnvironment"]
|
server/app.py
ADDED
|
@@ -0,0 +1,276 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
OpenOps FastAPI Server
|
| 3 |
+
Provides REST API endpoints for the incident management environment
|
| 4 |
+
"""
|
| 5 |
+
|
| 6 |
+
import sys
|
| 7 |
+
from pathlib import Path
|
| 8 |
+
|
| 9 |
+
# Add parent directory to path
|
| 10 |
+
sys.path.insert(0, str(Path(__file__).parent.parent))
|
| 11 |
+
|
| 12 |
+
from fastapi import FastAPI, HTTPException
|
| 13 |
+
from fastapi.responses import JSONResponse
|
| 14 |
+
from pydantic import BaseModel
|
| 15 |
+
from typing import Dict, Any, Optional
|
| 16 |
+
import uvicorn
|
| 17 |
+
|
| 18 |
+
from server.my_env_environment import MyEnvEnvironment
|
| 19 |
+
from models import IncidentAction, IncidentObservation
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
# FastAPI app
|
| 23 |
+
app = FastAPI(
|
| 24 |
+
title="OpenOps API",
|
| 25 |
+
description="Production Incident Management Environment API",
|
| 26 |
+
version="1.0.0"
|
| 27 |
+
)
|
| 28 |
+
|
| 29 |
+
# Global environment instance (stateful for demo purposes)
|
| 30 |
+
env_instance: Optional[MyEnvEnvironment] = None
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
# Request/Response Models
|
| 34 |
+
class ResetRequest(BaseModel):
|
| 35 |
+
task_id: int = 1
|
| 36 |
+
|
| 37 |
+
class Config:
|
| 38 |
+
json_schema_extra = {
|
| 39 |
+
"example": {
|
| 40 |
+
"task_id": 1
|
| 41 |
+
}
|
| 42 |
+
}
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
class StepRequest(BaseModel):
|
| 46 |
+
action_id: int
|
| 47 |
+
task_id: int = 1
|
| 48 |
+
|
| 49 |
+
class Config:
|
| 50 |
+
json_schema_extra = {
|
| 51 |
+
"example": {
|
| 52 |
+
"action_id": 0,
|
| 53 |
+
"task_id": 1
|
| 54 |
+
}
|
| 55 |
+
}
|
| 56 |
+
|
| 57 |
+
|
| 58 |
+
class StateResponse(BaseModel):
|
| 59 |
+
state: Dict[str, Any]
|
| 60 |
+
|
| 61 |
+
|
| 62 |
+
# Health check endpoint
|
| 63 |
+
@app.get("/")
|
| 64 |
+
async def root():
|
| 65 |
+
"""Health check endpoint."""
|
| 66 |
+
return {
|
| 67 |
+
"status": "healthy",
|
| 68 |
+
"service": "OpenOps Incident Commander",
|
| 69 |
+
"version": "1.0.0",
|
| 70 |
+
"endpoints": {
|
| 71 |
+
"docs": "/docs",
|
| 72 |
+
"reset": "/reset",
|
| 73 |
+
"step": "/step",
|
| 74 |
+
"state": "/state"
|
| 75 |
+
}
|
| 76 |
+
}
|
| 77 |
+
|
| 78 |
+
|
| 79 |
+
@app.get("/health")
|
| 80 |
+
async def health():
|
| 81 |
+
"""Detailed health check."""
|
| 82 |
+
return {
|
| 83 |
+
"status": "healthy",
|
| 84 |
+
"environment_loaded": env_instance is not None,
|
| 85 |
+
"current_task": env_instance.task_id if env_instance else None
|
| 86 |
+
}
|
| 87 |
+
|
| 88 |
+
|
| 89 |
+
@app.post("/reset")
|
| 90 |
+
async def reset(request: ResetRequest) -> Dict[str, Any]:
|
| 91 |
+
"""
|
| 92 |
+
Reset the environment for a specific task.
|
| 93 |
+
|
| 94 |
+
Args:
|
| 95 |
+
request: ResetRequest with task_id (1=easy, 2=medium, 3=hard)
|
| 96 |
+
|
| 97 |
+
Returns:
|
| 98 |
+
Initial observation after reset
|
| 99 |
+
"""
|
| 100 |
+
global env_instance
|
| 101 |
+
|
| 102 |
+
try:
|
| 103 |
+
# Validate task_id
|
| 104 |
+
if request.task_id not in [1, 2, 3]:
|
| 105 |
+
raise HTTPException(
|
| 106 |
+
status_code=400,
|
| 107 |
+
detail=f"Invalid task_id: {request.task_id}. Must be 1, 2, or 3."
|
| 108 |
+
)
|
| 109 |
+
|
| 110 |
+
# Create new environment instance
|
| 111 |
+
env_instance = MyEnvEnvironment()
|
| 112 |
+
obs = env_instance.reset(task_id=request.task_id)
|
| 113 |
+
|
| 114 |
+
# Return observation as dict
|
| 115 |
+
return {
|
| 116 |
+
"observation": obs.model_dump(),
|
| 117 |
+
"task_id": request.task_id,
|
| 118 |
+
"message": "Environment reset successfully"
|
| 119 |
+
}
|
| 120 |
+
|
| 121 |
+
except Exception as e:
|
| 122 |
+
raise HTTPException(
|
| 123 |
+
status_code=500,
|
| 124 |
+
detail=f"Failed to reset environment: {str(e)}"
|
| 125 |
+
)
|
| 126 |
+
|
| 127 |
+
|
| 128 |
+
@app.post("/step")
|
| 129 |
+
async def step(request: StepRequest) -> Dict[str, Any]:
|
| 130 |
+
"""
|
| 131 |
+
Execute an action in the environment.
|
| 132 |
+
|
| 133 |
+
Args:
|
| 134 |
+
request: StepRequest with action_id and task_id
|
| 135 |
+
|
| 136 |
+
Returns:
|
| 137 |
+
Observation after taking the action
|
| 138 |
+
"""
|
| 139 |
+
global env_instance
|
| 140 |
+
|
| 141 |
+
try:
|
| 142 |
+
# Check if environment is initialized
|
| 143 |
+
if env_instance is None:
|
| 144 |
+
raise HTTPException(
|
| 145 |
+
status_code=400,
|
| 146 |
+
detail="Environment not initialized. Call /reset first."
|
| 147 |
+
)
|
| 148 |
+
|
| 149 |
+
# Validate action_id
|
| 150 |
+
if request.action_id < 0 or request.action_id > 20:
|
| 151 |
+
raise HTTPException(
|
| 152 |
+
status_code=400,
|
| 153 |
+
detail=f"Invalid action_id: {request.action_id}. Must be 0-20."
|
| 154 |
+
)
|
| 155 |
+
|
| 156 |
+
# Create action
|
| 157 |
+
action = IncidentAction(
|
| 158 |
+
action_id=request.action_id,
|
| 159 |
+
task_id=request.task_id
|
| 160 |
+
)
|
| 161 |
+
|
| 162 |
+
# Execute step
|
| 163 |
+
obs = env_instance.step(action)
|
| 164 |
+
|
| 165 |
+
# Get action name
|
| 166 |
+
action_name = env_instance.ACTION_NAMES.get(request.action_id, "unknown")
|
| 167 |
+
|
| 168 |
+
return {
|
| 169 |
+
"observation": obs.model_dump(),
|
| 170 |
+
"action_taken": {
|
| 171 |
+
"action_id": request.action_id,
|
| 172 |
+
"action_name": action_name
|
| 173 |
+
},
|
| 174 |
+
"reward": obs.reward,
|
| 175 |
+
"done": obs.done,
|
| 176 |
+
"total_reward": env_instance.total_reward,
|
| 177 |
+
"incident_resolved": env_instance.incident_resolved
|
| 178 |
+
}
|
| 179 |
+
|
| 180 |
+
except Exception as e:
|
| 181 |
+
raise HTTPException(
|
| 182 |
+
status_code=500,
|
| 183 |
+
detail=f"Failed to execute step: {str(e)}"
|
| 184 |
+
)
|
| 185 |
+
|
| 186 |
+
|
| 187 |
+
@app.get("/state")
|
| 188 |
+
async def get_state() -> Dict[str, Any]:
|
| 189 |
+
"""
|
| 190 |
+
Get current environment state.
|
| 191 |
+
|
| 192 |
+
Returns:
|
| 193 |
+
Current state of the environment
|
| 194 |
+
"""
|
| 195 |
+
global env_instance
|
| 196 |
+
|
| 197 |
+
try:
|
| 198 |
+
if env_instance is None:
|
| 199 |
+
raise HTTPException(
|
| 200 |
+
status_code=400,
|
| 201 |
+
detail="Environment not initialized. Call /reset first."
|
| 202 |
+
)
|
| 203 |
+
|
| 204 |
+
state = env_instance.state
|
| 205 |
+
|
| 206 |
+
return {
|
| 207 |
+
"state": state.model_dump() if hasattr(state, 'model_dump') else state,
|
| 208 |
+
"task_id": env_instance.task_id,
|
| 209 |
+
"total_reward": env_instance.total_reward,
|
| 210 |
+
"incident_resolved": env_instance.incident_resolved,
|
| 211 |
+
"time_elapsed": env_instance.time_elapsed
|
| 212 |
+
}
|
| 213 |
+
|
| 214 |
+
except Exception as e:
|
| 215 |
+
raise HTTPException(
|
| 216 |
+
status_code=500,
|
| 217 |
+
detail=f"Failed to get state: {str(e)}"
|
| 218 |
+
)
|
| 219 |
+
|
| 220 |
+
|
| 221 |
+
@app.get("/actions")
|
| 222 |
+
async def get_actions() -> Dict[str, Any]:
|
| 223 |
+
"""
|
| 224 |
+
Get list of available actions.
|
| 225 |
+
|
| 226 |
+
Returns:
|
| 227 |
+
Dictionary of action IDs and names
|
| 228 |
+
"""
|
| 229 |
+
try:
|
| 230 |
+
# Create temporary instance to get action names
|
| 231 |
+
temp_env = MyEnvEnvironment()
|
| 232 |
+
|
| 233 |
+
return {
|
| 234 |
+
"actions": temp_env.ACTION_NAMES,
|
| 235 |
+
"total_actions": len(temp_env.ACTION_NAMES)
|
| 236 |
+
}
|
| 237 |
+
|
| 238 |
+
except Exception as e:
|
| 239 |
+
raise HTTPException(
|
| 240 |
+
status_code=500,
|
| 241 |
+
detail=f"Failed to get actions: {str(e)}"
|
| 242 |
+
)
|
| 243 |
+
|
| 244 |
+
|
| 245 |
+
# Error handlers
|
| 246 |
+
@app.exception_handler(HTTPException)
|
| 247 |
+
async def http_exception_handler(request, exc):
|
| 248 |
+
"""Handle HTTP exceptions."""
|
| 249 |
+
return JSONResponse(
|
| 250 |
+
status_code=exc.status_code,
|
| 251 |
+
content={
|
| 252 |
+
"error": exc.detail,
|
| 253 |
+
"status_code": exc.status_code
|
| 254 |
+
}
|
| 255 |
+
)
|
| 256 |
+
|
| 257 |
+
|
| 258 |
+
@app.exception_handler(Exception)
|
| 259 |
+
async def general_exception_handler(request, exc):
|
| 260 |
+
"""Handle general exceptions."""
|
| 261 |
+
return JSONResponse(
|
| 262 |
+
status_code=500,
|
| 263 |
+
content={
|
| 264 |
+
"error": "Internal server error",
|
| 265 |
+
"detail": str(exc)
|
| 266 |
+
}
|
| 267 |
+
)
|
| 268 |
+
|
| 269 |
+
|
| 270 |
+
if __name__ == "__main__":
|
| 271 |
+
uvicorn.run(
|
| 272 |
+
app,
|
| 273 |
+
host="0.0.0.0",
|
| 274 |
+
port=7860,
|
| 275 |
+
log_level="info"
|
| 276 |
+
)
|
server/my_env_environment.py
ADDED
|
@@ -0,0 +1,409 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
OpenOps Production Incident Management Environment
|
| 3 |
+
Simulates real-world production incidents where agents must investigate, mitigate, and resolve issues
|
| 4 |
+
"""
|
| 5 |
+
|
| 6 |
+
from typing import Dict, List, Any, Optional
|
| 7 |
+
from openenv.core import Environment
|
| 8 |
+
from models import IncidentAction, IncidentObservation, IncidentState
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
class MyEnvEnvironment(Environment):
|
| 12 |
+
"""
|
| 13 |
+
Production incident management environment.
|
| 14 |
+
|
| 15 |
+
Simulates 3 types of incidents:
|
| 16 |
+
- Task 1 (Easy): Simple API crash requiring restart
|
| 17 |
+
- Task 2 (Medium): Bad deployment requiring rollback
|
| 18 |
+
- Task 3 (Hard): Cascading database overload requiring multi-step resolution
|
| 19 |
+
"""
|
| 20 |
+
|
| 21 |
+
# Action definitions
|
| 22 |
+
ACTION_NAMES = {
|
| 23 |
+
# Investigation actions (0-8)
|
| 24 |
+
0: "read_alerts",
|
| 25 |
+
1: "inspect_logs_api",
|
| 26 |
+
2: "inspect_logs_database",
|
| 27 |
+
3: "inspect_logs_auth",
|
| 28 |
+
4: "inspect_logs_frontend",
|
| 29 |
+
5: "check_metrics_api",
|
| 30 |
+
6: "check_metrics_database",
|
| 31 |
+
7: "check_metrics_auth",
|
| 32 |
+
8: "check_metrics_frontend",
|
| 33 |
+
|
| 34 |
+
# Mitigation actions (9-16)
|
| 35 |
+
9: "restart_api",
|
| 36 |
+
10: "restart_database",
|
| 37 |
+
11: "restart_auth",
|
| 38 |
+
12: "restart_frontend",
|
| 39 |
+
13: "rollback_api",
|
| 40 |
+
14: "rollback_database",
|
| 41 |
+
15: "scale_api",
|
| 42 |
+
16: "scale_database",
|
| 43 |
+
|
| 44 |
+
# Communication actions (17-19)
|
| 45 |
+
17: "notify_team",
|
| 46 |
+
18: "update_status_page",
|
| 47 |
+
19: "send_user_communication",
|
| 48 |
+
|
| 49 |
+
# Resolution (20)
|
| 50 |
+
20: "resolve_incident"
|
| 51 |
+
}
|
| 52 |
+
|
| 53 |
+
def __init__(self):
|
| 54 |
+
"""Initialize the environment."""
|
| 55 |
+
super().__init__()
|
| 56 |
+
self.task_id = 1
|
| 57 |
+
self.time_elapsed = 0
|
| 58 |
+
self.max_steps = 30
|
| 59 |
+
self.total_reward = 0.0
|
| 60 |
+
|
| 61 |
+
# State tracking
|
| 62 |
+
self.incident_resolved = False
|
| 63 |
+
self.alerts_read = False
|
| 64 |
+
self.logs_inspected = set()
|
| 65 |
+
self.metrics_checked = set()
|
| 66 |
+
self.services_restarted = set()
|
| 67 |
+
self.services_rolled_back = set()
|
| 68 |
+
self.services_scaled = set()
|
| 69 |
+
self.teams_notified = False
|
| 70 |
+
self.status_page_updated = False
|
| 71 |
+
self.users_communicated = False
|
| 72 |
+
|
| 73 |
+
# Internal state
|
| 74 |
+
self._state = None
|
| 75 |
+
|
| 76 |
+
@property
|
| 77 |
+
def state(self) -> IncidentState:
|
| 78 |
+
"""
|
| 79 |
+
Return current environment state.
|
| 80 |
+
Required by BaseEnvironment abstract class.
|
| 81 |
+
"""
|
| 82 |
+
return self._state
|
| 83 |
+
|
| 84 |
+
@state.setter
|
| 85 |
+
def state(self, value: IncidentState):
|
| 86 |
+
"""Set the environment state."""
|
| 87 |
+
self._state = value
|
| 88 |
+
|
| 89 |
+
def reset(self, task_id: int = 1) -> IncidentObservation:
|
| 90 |
+
"""
|
| 91 |
+
Reset environment for a specific task.
|
| 92 |
+
|
| 93 |
+
Args:
|
| 94 |
+
task_id: Task difficulty (1=easy, 2=medium, 3=hard)
|
| 95 |
+
|
| 96 |
+
Returns:
|
| 97 |
+
Initial observation
|
| 98 |
+
"""
|
| 99 |
+
self.task_id = task_id
|
| 100 |
+
self.time_elapsed = 0
|
| 101 |
+
self.total_reward = 0.0
|
| 102 |
+
|
| 103 |
+
# Reset tracking
|
| 104 |
+
self.incident_resolved = False
|
| 105 |
+
self.alerts_read = False
|
| 106 |
+
self.logs_inspected = set()
|
| 107 |
+
self.metrics_checked = set()
|
| 108 |
+
self.services_restarted = set()
|
| 109 |
+
self.services_rolled_back = set()
|
| 110 |
+
self.services_scaled = set()
|
| 111 |
+
self.teams_notified = False
|
| 112 |
+
self.status_page_updated = False
|
| 113 |
+
self.users_communicated = False
|
| 114 |
+
|
| 115 |
+
# Initialize state based on task
|
| 116 |
+
self._state = self._init_task_state(task_id)
|
| 117 |
+
|
| 118 |
+
# Return initial observation
|
| 119 |
+
return self._get_observation()
|
| 120 |
+
|
| 121 |
+
def _init_task_state(self, task_id: int) -> IncidentState:
|
| 122 |
+
"""Initialize task-specific state."""
|
| 123 |
+
|
| 124 |
+
if task_id == 1:
|
| 125 |
+
# Task 1: Simple API crash (OOM)
|
| 126 |
+
return IncidentState(
|
| 127 |
+
task_id=task_id,
|
| 128 |
+
incident_type="api_crash",
|
| 129 |
+
affected_services=["api"],
|
| 130 |
+
root_cause="out_of_memory",
|
| 131 |
+
service_status={
|
| 132 |
+
"api": "down",
|
| 133 |
+
"database": "healthy",
|
| 134 |
+
"auth": "healthy",
|
| 135 |
+
"frontend": "degraded"
|
| 136 |
+
},
|
| 137 |
+
correct_mitigation=["restart_api"],
|
| 138 |
+
revenue_loss=0.0,
|
| 139 |
+
customer_complaints=0
|
| 140 |
+
)
|
| 141 |
+
|
| 142 |
+
elif task_id == 2:
|
| 143 |
+
# Task 2: Bad deployment (database)
|
| 144 |
+
return IncidentState(
|
| 145 |
+
task_id=task_id,
|
| 146 |
+
incident_type="bad_deployment",
|
| 147 |
+
affected_services=["database", "api"],
|
| 148 |
+
root_cause="bad_migration",
|
| 149 |
+
service_status={
|
| 150 |
+
"api": "degraded",
|
| 151 |
+
"database": "degraded",
|
| 152 |
+
"auth": "healthy",
|
| 153 |
+
"frontend": "degraded"
|
| 154 |
+
},
|
| 155 |
+
correct_mitigation=["rollback_database"],
|
| 156 |
+
revenue_loss=0.0,
|
| 157 |
+
customer_complaints=0
|
| 158 |
+
)
|
| 159 |
+
|
| 160 |
+
else: # task_id == 3
|
| 161 |
+
# Task 3: Cascading failure (database overload)
|
| 162 |
+
return IncidentState(
|
| 163 |
+
task_id=task_id,
|
| 164 |
+
incident_type="cascading_failure",
|
| 165 |
+
affected_services=["database", "api"],
|
| 166 |
+
root_cause="database_overload",
|
| 167 |
+
service_status={
|
| 168 |
+
"api": "degraded",
|
| 169 |
+
"database": "degraded",
|
| 170 |
+
"auth": "healthy",
|
| 171 |
+
"frontend": "degraded"
|
| 172 |
+
},
|
| 173 |
+
correct_mitigation=["scale_database", "restart_api"],
|
| 174 |
+
revenue_loss=0.0,
|
| 175 |
+
customer_complaints=0
|
| 176 |
+
)
|
| 177 |
+
|
| 178 |
+
def step(self, action: IncidentAction) -> IncidentObservation:
|
| 179 |
+
"""
|
| 180 |
+
Execute an action and return observation.
|
| 181 |
+
|
| 182 |
+
Args:
|
| 183 |
+
action: Action to execute
|
| 184 |
+
|
| 185 |
+
Returns:
|
| 186 |
+
Observation after action execution
|
| 187 |
+
"""
|
| 188 |
+
self.time_elapsed += 1
|
| 189 |
+
reward = 0.0
|
| 190 |
+
done = False
|
| 191 |
+
|
| 192 |
+
# Time penalty
|
| 193 |
+
reward -= 0.05
|
| 194 |
+
|
| 195 |
+
# Revenue loss increases over time
|
| 196 |
+
self._state.revenue_loss += 1000 * self.time_elapsed
|
| 197 |
+
self._state.customer_complaints += self.time_elapsed // 3
|
| 198 |
+
|
| 199 |
+
# Execute action
|
| 200 |
+
action_name = self.ACTION_NAMES.get(action.action_id, "unknown")
|
| 201 |
+
|
| 202 |
+
# Investigation actions
|
| 203 |
+
if action.action_id == 0: # read_alerts
|
| 204 |
+
if not self.alerts_read:
|
| 205 |
+
self.alerts_read = True
|
| 206 |
+
reward += 0.05
|
| 207 |
+
|
| 208 |
+
elif 1 <= action.action_id <= 4: # inspect_logs
|
| 209 |
+
service = ["api", "database", "auth", "frontend"][action.action_id - 1]
|
| 210 |
+
if service not in self.logs_inspected:
|
| 211 |
+
self.logs_inspected.add(service)
|
| 212 |
+
if service in self._state.affected_services:
|
| 213 |
+
reward += 0.25 # Bonus for inspecting affected service
|
| 214 |
+
else:
|
| 215 |
+
reward += 0.05
|
| 216 |
+
|
| 217 |
+
elif 5 <= action.action_id <= 8: # check_metrics
|
| 218 |
+
service = ["api", "database", "auth", "frontend"][action.action_id - 5]
|
| 219 |
+
if service not in self.metrics_checked:
|
| 220 |
+
self.metrics_checked.add(service)
|
| 221 |
+
reward += 0.05
|
| 222 |
+
|
| 223 |
+
# Mitigation actions
|
| 224 |
+
elif 9 <= action.action_id <= 12: # restart services
|
| 225 |
+
service = ["api", "database", "auth", "frontend"][action.action_id - 9]
|
| 226 |
+
if service not in self.services_restarted:
|
| 227 |
+
self.services_restarted.add(service)
|
| 228 |
+
|
| 229 |
+
# Check if restart is correct mitigation
|
| 230 |
+
if "restart_" + service in self._state.correct_mitigation:
|
| 231 |
+
reward += 0.75
|
| 232 |
+
self._state.service_status[service] = "healthy"
|
| 233 |
+
elif service in self._state.affected_services:
|
| 234 |
+
# Restarting affected service (but not the solution)
|
| 235 |
+
reward -= 0.5
|
| 236 |
+
else:
|
| 237 |
+
reward -= 0.1
|
| 238 |
+
|
| 239 |
+
elif 13 <= action.action_id <= 14: # rollback (API or Database)
|
| 240 |
+
service = ["api", "database"][action.action_id - 13]
|
| 241 |
+
if service not in self.services_rolled_back:
|
| 242 |
+
self.services_rolled_back.add(service)
|
| 243 |
+
|
| 244 |
+
# Check if rollback is correct
|
| 245 |
+
if "rollback_" + service in self._state.correct_mitigation:
|
| 246 |
+
reward += 1.0
|
| 247 |
+
self._state.service_status[service] = "healthy"
|
| 248 |
+
if service == "database":
|
| 249 |
+
self._state.service_status["api"] = "healthy" # Fixes downstream
|
| 250 |
+
else:
|
| 251 |
+
reward -= 0.3
|
| 252 |
+
|
| 253 |
+
elif 15 <= action.action_id <= 16: # scale (API or Database)
|
| 254 |
+
service = ["api", "database"][action.action_id - 15]
|
| 255 |
+
if service not in self.services_scaled:
|
| 256 |
+
self.services_scaled.add(service)
|
| 257 |
+
|
| 258 |
+
# Check if scaling is correct
|
| 259 |
+
if "scale_" + service in self._state.correct_mitigation:
|
| 260 |
+
reward += 0.75
|
| 261 |
+
self._state.service_status[service] = "healthy"
|
| 262 |
+
else:
|
| 263 |
+
reward -= 0.2
|
| 264 |
+
|
| 265 |
+
# Communication actions
|
| 266 |
+
elif action.action_id == 17: # notify_team
|
| 267 |
+
if not self.teams_notified:
|
| 268 |
+
self.teams_notified = True
|
| 269 |
+
reward += 0.25
|
| 270 |
+
|
| 271 |
+
elif action.action_id == 18: # update_status_page
|
| 272 |
+
if not self.status_page_updated:
|
| 273 |
+
self.status_page_updated = True
|
| 274 |
+
reward += 0.25
|
| 275 |
+
|
| 276 |
+
elif action.action_id == 19: # send_user_communication
|
| 277 |
+
if not self.users_communicated:
|
| 278 |
+
self.users_communicated = True
|
| 279 |
+
reward += 0.15
|
| 280 |
+
|
| 281 |
+
# Resolution
|
| 282 |
+
elif action.action_id == 20: # resolve_incident
|
| 283 |
+
# Check if all services are healthy
|
| 284 |
+
all_healthy = all(
|
| 285 |
+
status == "healthy"
|
| 286 |
+
for service, status in self._state.service_status.items()
|
| 287 |
+
if service in self._state.affected_services
|
| 288 |
+
)
|
| 289 |
+
|
| 290 |
+
if all_healthy:
|
| 291 |
+
self.incident_resolved = True
|
| 292 |
+
# Big reward for resolution
|
| 293 |
+
reward += 3.0
|
| 294 |
+
# Time bonus (faster = better)
|
| 295 |
+
time_bonus = max(0, (30 - self.time_elapsed) * 0.01)
|
| 296 |
+
reward += time_bonus
|
| 297 |
+
done = True
|
| 298 |
+
else:
|
| 299 |
+
# Penalty for premature resolution
|
| 300 |
+
reward -= 1.0
|
| 301 |
+
done = True
|
| 302 |
+
|
| 303 |
+
# Update total reward
|
| 304 |
+
self.total_reward += reward
|
| 305 |
+
|
| 306 |
+
# Check timeout
|
| 307 |
+
if self.time_elapsed >= self.max_steps:
|
| 308 |
+
done = True
|
| 309 |
+
|
| 310 |
+
# Return observation
|
| 311 |
+
obs = self._get_observation()
|
| 312 |
+
obs.reward = reward
|
| 313 |
+
obs.done = done
|
| 314 |
+
|
| 315 |
+
return obs
|
| 316 |
+
|
| 317 |
+
def _get_observation(self) -> IncidentObservation:
|
| 318 |
+
"""Generate current observation."""
|
| 319 |
+
|
| 320 |
+
# Build alerts
|
| 321 |
+
active_alerts = []
|
| 322 |
+
if not self.alerts_read:
|
| 323 |
+
active_alerts = ["[Call read_alerts to see alerts]"]
|
| 324 |
+
else:
|
| 325 |
+
if self._state.task_id == 1:
|
| 326 |
+
active_alerts = [
|
| 327 |
+
"๐จ CRITICAL: API service down - no response",
|
| 328 |
+
"โ ๏ธ HIGH: Frontend experiencing errors",
|
| 329 |
+
"๐ Customer complaints spiking"
|
| 330 |
+
]
|
| 331 |
+
elif self._state.task_id == 2:
|
| 332 |
+
active_alerts = [
|
| 333 |
+
"๐จ CRITICAL: Database queries failing",
|
| 334 |
+
"โ ๏ธ HIGH: API returning 500 errors",
|
| 335 |
+
"๐ Recent deployment detected"
|
| 336 |
+
]
|
| 337 |
+
else: # task_id == 3
|
| 338 |
+
active_alerts = [
|
| 339 |
+
"๐จ CRITICAL: Database CPU at 95%",
|
| 340 |
+
"๐จ CRITICAL: API timeout rate high",
|
| 341 |
+
"โ ๏ธ HIGH: Connection pool exhausted",
|
| 342 |
+
"๐ Cascading failure detected"
|
| 343 |
+
]
|
| 344 |
+
|
| 345 |
+
# Build logs (only for inspected services)
|
| 346 |
+
recent_logs = {}
|
| 347 |
+
for service in self.logs_inspected:
|
| 348 |
+
if self._state.task_id == 1 and service == "api":
|
| 349 |
+
recent_logs["api"] = [
|
| 350 |
+
"ERROR: Out of memory - process killed",
|
| 351 |
+
"INFO: Last request before crash at 14:32:15"
|
| 352 |
+
]
|
| 353 |
+
elif self._state.task_id == 2 and service == "database":
|
| 354 |
+
recent_logs["database"] = [
|
| 355 |
+
"ERROR: Syntax error in migration v2.3.1",
|
| 356 |
+
"ERROR: Incompatible schema changes detected"
|
| 357 |
+
]
|
| 358 |
+
elif self._state.task_id == 2 and service == "api":
|
| 359 |
+
recent_logs["api"] = [
|
| 360 |
+
"ERROR: Database query timeout",
|
| 361 |
+
"ERROR: 500 Internal Server Error"
|
| 362 |
+
]
|
| 363 |
+
elif self._state.task_id == 3 and service == "database":
|
| 364 |
+
recent_logs["database"] = [
|
| 365 |
+
"WARN: Connection pool exhausted (95% utilization)",
|
| 366 |
+
"ERROR: Slow query detected (>10s)",
|
| 367 |
+
"WARN: CPU usage at 95%"
|
| 368 |
+
]
|
| 369 |
+
elif self._state.task_id == 3 and service == "api":
|
| 370 |
+
recent_logs["api"] = [
|
| 371 |
+
"ERROR: Database connection timeout",
|
| 372 |
+
"ERROR: Request timeout (30s exceeded)"
|
| 373 |
+
]
|
| 374 |
+
|
| 375 |
+
# Build metrics summary
|
| 376 |
+
metrics_summary = {}
|
| 377 |
+
for service in self.metrics_checked:
|
| 378 |
+
if service in self._state.affected_services:
|
| 379 |
+
metrics_summary[service] = {
|
| 380 |
+
"cpu": 85.0 if service == "database" else 45.0,
|
| 381 |
+
"memory": 92.0 if service == "api" else 60.0,
|
| 382 |
+
"latency": 5000.0 if service in ["api", "database"] else 100.0
|
| 383 |
+
}
|
| 384 |
+
|
| 385 |
+
return IncidentObservation(
|
| 386 |
+
active_alerts=active_alerts,
|
| 387 |
+
service_status=self._state.service_status.copy(),
|
| 388 |
+
recent_logs=recent_logs,
|
| 389 |
+
metrics_summary=metrics_summary,
|
| 390 |
+
customer_complaints=self._state.customer_complaints,
|
| 391 |
+
time_elapsed=self.time_elapsed,
|
| 392 |
+
revenue_loss=self._state.revenue_loss,
|
| 393 |
+
teams_notified=self.teams_notified,
|
| 394 |
+
status_page_updated=self.status_page_updated,
|
| 395 |
+
reward=0.0,
|
| 396 |
+
done=False
|
| 397 |
+
)
|
| 398 |
+
|
| 399 |
+
def render(self):
|
| 400 |
+
"""Render current state (optional for debugging)."""
|
| 401 |
+
print(f"\n{'='*60}")
|
| 402 |
+
print(f"Task {self.task_id} - Step {self.time_elapsed}")
|
| 403 |
+
print(f"{'='*60}")
|
| 404 |
+
print(f"Service Status: {self._state.service_status}")
|
| 405 |
+
print(f"Revenue Loss: ${self._state.revenue_loss:,.0f}")
|
| 406 |
+
print(f"Complaints: {self._state.customer_complaints}")
|
| 407 |
+
print(f"Incident Resolved: {self.incident_resolved}")
|
| 408 |
+
print(f"Total Reward: {self.total_reward:.2f}")
|
| 409 |
+
print(f"{'='*60}\n")
|
server/requirements.txt
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
fastapi>=0.115.0
|
| 2 |
+
uvicorn[standard]>=0.32.0
|
| 3 |
+
pydantic>=2.0.0
|
| 4 |
+
openai>=1.0.0
|
| 5 |
+
openenv-core>=0.2.1
|
| 6 |
+
python-multipart>=0.0.9
|
submission_log.txt
ADDED
|
Binary file (6.31 kB). View file
|
|
|
test_env.py
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from server.my_env_environment import MyEnvEnvironment
|
| 2 |
+
from models import IncidentAction
|
| 3 |
+
from graders import get_grader
|
| 4 |
+
|
| 5 |
+
env = MyEnvEnvironment()
|
| 6 |
+
obs = env.reset(task_id=1)
|
| 7 |
+
print("Initial observation:", obs.active_alerts)
|
| 8 |
+
|
| 9 |
+
# Take some actions
|
| 10 |
+
obs = env.step(IncidentAction(action_id=0, task_id=1)) # read_alerts
|
| 11 |
+
print("After read_alerts:", obs.reward)
|
| 12 |
+
|
| 13 |
+
obs = env.step(IncidentAction(action_id=1, task_id=1)) # inspect_logs_api
|
| 14 |
+
print("After inspect_logs:", obs.reward)
|
| 15 |
+
|
| 16 |
+
obs = env.step(IncidentAction(action_id=9, task_id=1)) # restart_api
|
| 17 |
+
print("After restart:", obs.reward)
|
| 18 |
+
|
| 19 |
+
obs = env.step(IncidentAction(action_id=20, task_id=1)) # resolve
|
| 20 |
+
print("After resolve:", obs.reward, obs.done)
|
| 21 |
+
|
| 22 |
+
grader = get_grader(1)
|
| 23 |
+
score = grader(env)
|
| 24 |
+
print("Final score:", score)
|
test_local.py
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Quick local test - runs in <5 seconds"""
|
| 2 |
+
from server.my_env_environment import MyEnvEnvironment
|
| 3 |
+
from models import IncidentAction
|
| 4 |
+
from graders import get_grader
|
| 5 |
+
|
| 6 |
+
print("Testing OpenOps environment...")
|
| 7 |
+
|
| 8 |
+
for task_id in [1, 2, 3]:
|
| 9 |
+
env = MyEnvEnvironment()
|
| 10 |
+
env.reset(task_id=task_id)
|
| 11 |
+
|
| 12 |
+
# Take a few actions
|
| 13 |
+
env.step(IncidentAction(action_id=0, task_id=task_id))
|
| 14 |
+
env.step(IncidentAction(action_id=1, task_id=task_id))
|
| 15 |
+
|
| 16 |
+
grader = get_grader(task_id)
|
| 17 |
+
score = grader(env)
|
| 18 |
+
print(f"Task {task_id}: Environment working โ
(test score: {score:.2f})")
|
| 19 |
+
|
| 20 |
+
print("\nโ
All tests passed!")
|
validate_submission.py
ADDED
|
@@ -0,0 +1,132 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Pre-submission validation - Run this before deploying!
|
| 3 |
+
"""
|
| 4 |
+
|
| 5 |
+
import os
|
| 6 |
+
import sys
|
| 7 |
+
import subprocess
|
| 8 |
+
|
| 9 |
+
|
| 10 |
+
def check(condition, message):
|
| 11 |
+
"""Helper to check condition."""
|
| 12 |
+
if condition:
|
| 13 |
+
print(f"โ
{message}")
|
| 14 |
+
return True
|
| 15 |
+
else:
|
| 16 |
+
print(f"โ {message}")
|
| 17 |
+
return False
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
def validate():
|
| 21 |
+
"""Run all validation checks."""
|
| 22 |
+
print("="*60)
|
| 23 |
+
print("OpenOps Pre-Submission Validation")
|
| 24 |
+
print("="*60)
|
| 25 |
+
print()
|
| 26 |
+
|
| 27 |
+
checks = []
|
| 28 |
+
|
| 29 |
+
# File existence
|
| 30 |
+
print("๐ Checking required files...")
|
| 31 |
+
checks.append(check(os.path.exists("models.py"), "models.py exists"))
|
| 32 |
+
checks.append(check(os.path.exists("server/my_env_environment.py"), "server/my_env_environment.py exists"))
|
| 33 |
+
checks.append(check(os.path.exists("graders.py"), "graders.py exists"))
|
| 34 |
+
checks.append(check(os.path.exists("inference.py"), "inference.py exists"))
|
| 35 |
+
checks.append(check(os.path.exists("openenv.yaml"), "openenv.yaml exists"))
|
| 36 |
+
checks.append(check(os.path.exists("README.md"), "README.md exists"))
|
| 37 |
+
checks.append(check(os.path.exists("server/Dockerfile"), "server/Dockerfile exists"))
|
| 38 |
+
checks.append(check(os.path.exists("server/requirements.txt"), "server/requirements.txt exists"))
|
| 39 |
+
checks.append(check(os.path.exists("server/app.py"), "server/app.py exists"))
|
| 40 |
+
checks.append(check(os.path.exists("client.py"), "client.py exists"))
|
| 41 |
+
print()
|
| 42 |
+
|
| 43 |
+
# Import test
|
| 44 |
+
print("๐ง Testing imports...")
|
| 45 |
+
try:
|
| 46 |
+
from models import IncidentAction, IncidentObservation, IncidentState
|
| 47 |
+
from server.my_env_environment import MyEnvEnvironment
|
| 48 |
+
from graders import get_grader
|
| 49 |
+
checks.append(check(True, "All imports successful"))
|
| 50 |
+
except Exception as e:
|
| 51 |
+
checks.append(check(False, f"Import failed: {e}"))
|
| 52 |
+
print()
|
| 53 |
+
|
| 54 |
+
# Environment test
|
| 55 |
+
print("๐ฎ Testing environment...")
|
| 56 |
+
try:
|
| 57 |
+
from server.my_env_environment import MyEnvEnvironment
|
| 58 |
+
from models import IncidentAction
|
| 59 |
+
|
| 60 |
+
env = MyEnvEnvironment()
|
| 61 |
+
obs = env.reset(task_id=1)
|
| 62 |
+
checks.append(check(obs is not None, "Environment resets"))
|
| 63 |
+
|
| 64 |
+
action = IncidentAction(action_id=0, task_id=1)
|
| 65 |
+
obs = env.step(action)
|
| 66 |
+
checks.append(check(obs.reward is not None, "Environment steps"))
|
| 67 |
+
except Exception as e:
|
| 68 |
+
checks.append(check(False, f"Environment test failed: {e}"))
|
| 69 |
+
print()
|
| 70 |
+
|
| 71 |
+
# Grader test
|
| 72 |
+
print("๐ Testing graders...")
|
| 73 |
+
try:
|
| 74 |
+
from graders import get_grader
|
| 75 |
+
from server.my_env_environment import MyEnvEnvironment
|
| 76 |
+
|
| 77 |
+
env = MyEnvEnvironment()
|
| 78 |
+
env.reset(task_id=1)
|
| 79 |
+
grader = get_grader(1)
|
| 80 |
+
score = grader(env)
|
| 81 |
+
checks.append(check(0.0 <= score <= 1.0, f"Grader works (score: {score:.2f})"))
|
| 82 |
+
except Exception as e:
|
| 83 |
+
checks.append(check(False, f"Grader test failed: {e}"))
|
| 84 |
+
print()
|
| 85 |
+
|
| 86 |
+
# Inference script test
|
| 87 |
+
print("๐ค Testing inference script...")
|
| 88 |
+
try:
|
| 89 |
+
with open("inference.py", "r") as f:
|
| 90 |
+
content = f.read()
|
| 91 |
+
|
| 92 |
+
has_start = "[START]" in content
|
| 93 |
+
has_step = "[STEP]" in content
|
| 94 |
+
has_end = "[END]" in content
|
| 95 |
+
|
| 96 |
+
checks.append(check(has_start and has_step and has_end, "Has required log format"))
|
| 97 |
+
checks.append(check("from openai import OpenAI" in content or "OpenAI" in content, "Uses OpenAI-compatible client"))
|
| 98 |
+
except Exception as e:
|
| 99 |
+
checks.append(check(False, f"Inference validation failed: {e}"))
|
| 100 |
+
print()
|
| 101 |
+
|
| 102 |
+
# README check
|
| 103 |
+
print("๐ Checking README...")
|
| 104 |
+
try:
|
| 105 |
+
with open("README.md", "r") as f:
|
| 106 |
+
readme = f.read()
|
| 107 |
+
|
| 108 |
+
checks.append(check(len(readme) > 500, "README has content (>500 chars)"))
|
| 109 |
+
checks.append(check("OpenOps" in readme, "README mentions project name"))
|
| 110 |
+
checks.append(check("Task 1" in readme or "Task 2" in readme, "README describes tasks"))
|
| 111 |
+
except Exception as e:
|
| 112 |
+
checks.append(check(False, f"README check failed: {e}"))
|
| 113 |
+
print()
|
| 114 |
+
|
| 115 |
+
# Summary
|
| 116 |
+
print("="*60)
|
| 117 |
+
passed = sum(checks)
|
| 118 |
+
total = len(checks)
|
| 119 |
+
|
| 120 |
+
print(f"\nResults: {passed}/{total} checks passed")
|
| 121 |
+
|
| 122 |
+
if passed == total:
|
| 123 |
+
print("\nโ
ALL CHECKS PASSED - READY TO SUBMIT! ๐")
|
| 124 |
+
return True
|
| 125 |
+
else:
|
| 126 |
+
print(f"\nโ ๏ธ {total - passed} issues found - FIX BEFORE SUBMITTING!")
|
| 127 |
+
return False
|
| 128 |
+
|
| 129 |
+
|
| 130 |
+
if __name__ == "__main__":
|
| 131 |
+
success = validate()
|
| 132 |
+
sys.exit(0 if success else 1)
|