Spaces:
Sleeping
Sleeping
Upload folder using huggingface_hub
Browse filesThis view is limited to 50 files because it contains too many changes. See raw diff
- Dockerfile +41 -0
- README.md +78 -5
- __init__.py +11 -0
- client.py +100 -0
- comparison.md +96 -0
- generate_scenarios.py +1336 -0
- models.py +86 -0
- openenv.yaml +8 -0
- openenv_spreadsheet.egg-info/PKG-INFO +17 -0
- openenv_spreadsheet.egg-info/SOURCES.txt +24 -0
- openenv_spreadsheet.egg-info/dependency_links.txt +1 -0
- openenv_spreadsheet.egg-info/entry_points.txt +2 -0
- openenv_spreadsheet.egg-info/requires.txt +13 -0
- openenv_spreadsheet.egg-info/top_level.txt +1 -0
- pyproject.toml +37 -0
- scenarios/.gitkeep +0 -0
- scenarios/buggy_template_fix_01.json +8 -0
- scenarios/conditional_aggregation_01.json +8 -0
- scenarios/conditional_aggregation_02.json +8 -0
- scenarios/cross_sheet_lookup_01.json +8 -0
- scenarios/cross_sheet_lookup_02.json +8 -0
- scenarios/formula_repair_01.json +8 -0
- scenarios/formula_repair_02.json +8 -0
- scenarios/ledger_reconciliation_01.json +8 -0
- scenarios/ledger_reconciliation_02.json +8 -0
- scenarios/messy_table_extraction_01.json +8 -0
- scenarios/range_transformation_01.json +8 -0
- scenarios/schedule_grid_fill_01.json +8 -0
- server/__init__.py +11 -0
- server/app.py +41 -0
- server/formula_utils.py +39 -0
- server/scenario_loader.py +62 -0
- server/spreadsheet_environment.py +445 -0
- server/workbook_engine.py +564 -0
- spreadsheet.egg-info/PKG-INFO +16 -0
- spreadsheet.egg-info/SOURCES.txt +22 -0
- spreadsheet.egg-info/dependency_links.txt +1 -0
- spreadsheet.egg-info/entry_points.txt +2 -0
- spreadsheet.egg-info/requires.txt +13 -0
- spreadsheet.egg-info/top_level.txt +1 -0
- uv.lock +0 -0
- workbooks/fixtures/.gitkeep +0 -0
- workbooks/fixtures/037858b5-3d0e-4714-8640-2dea23fc3a18_multi_currency_reconciliation.xlsx +0 -0
- workbooks/fixtures/1333ba32-7957-4f7f-b310-6a9ba0e718bd_data_pivot_reshape.xlsx +0 -0
- workbooks/fixtures/15123e53-9510-48d4-ae1a-a01556145b8e_employee_bonus_calculation.xlsx +0 -0
- workbooks/fixtures/158cebfc-4813-49c4-bd54-fceef44c4860_employee_schedule_grid.xlsx +0 -0
- workbooks/fixtures/19d8a671-1769-45aa-af51-39d12e81d45c_multi_currency_reconciliation.xlsx +0 -0
- workbooks/fixtures/30f43287-34a1-4620-a9ae-3d982705a5e5_bank_reconciliation.xlsx +0 -0
- workbooks/fixtures/45bb730b-c042-491e-9f7d-ff9ff3de25a6_cascading_formula_errors.xlsx +0 -0
- workbooks/fixtures/5a43518a-7b4c-4e49-a2e3-ae0e550f5351_multi_department_budget.xlsx +0 -0
Dockerfile
ADDED
|
@@ -0,0 +1,41 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
ARG BASE_IMAGE=ghcr.io/meta-pytorch/openenv-base:latest
|
| 2 |
+
FROM ${BASE_IMAGE} AS builder
|
| 3 |
+
|
| 4 |
+
RUN apt-get update && \
|
| 5 |
+
apt-get install -y --no-install-recommends git curl && \
|
| 6 |
+
rm -rf /var/lib/apt/lists/*
|
| 7 |
+
|
| 8 |
+
WORKDIR /app
|
| 9 |
+
COPY . /app/env
|
| 10 |
+
WORKDIR /app/env
|
| 11 |
+
|
| 12 |
+
RUN if ! command -v uv >/dev/null 2>&1; then \
|
| 13 |
+
curl -LsSf https://astral.sh/uv/install.sh | sh && \
|
| 14 |
+
mv /root/.local/bin/uv /usr/local/bin/uv; \
|
| 15 |
+
fi
|
| 16 |
+
|
| 17 |
+
RUN --mount=type=cache,target=/root/.cache/uv \
|
| 18 |
+
if [ -f uv.lock ]; then uv sync --frozen --no-install-project --no-editable; \
|
| 19 |
+
else uv sync --no-install-project --no-editable; fi
|
| 20 |
+
|
| 21 |
+
RUN --mount=type=cache,target=/root/.cache/uv \
|
| 22 |
+
if [ -f uv.lock ]; then uv sync --frozen --no-editable; \
|
| 23 |
+
else uv sync --no-editable; fi
|
| 24 |
+
|
| 25 |
+
FROM ${BASE_IMAGE}
|
| 26 |
+
WORKDIR /app
|
| 27 |
+
COPY --from=builder /app/env/.venv /app/.venv
|
| 28 |
+
COPY --from=builder /app/env /app/env
|
| 29 |
+
|
| 30 |
+
ENV PATH="/app/.venv/bin:$PATH"
|
| 31 |
+
ENV PYTHONPATH="/app/env:$PYTHONPATH"
|
| 32 |
+
ENV ENABLE_WEB_INTERFACE=true
|
| 33 |
+
ENV WORKBOOKS_DIR=/app/env/workbooks
|
| 34 |
+
ENV SCENARIOS_DIR=/app/env/scenarios
|
| 35 |
+
|
| 36 |
+
EXPOSE 8000
|
| 37 |
+
|
| 38 |
+
HEALTHCHECK --interval=30s --timeout=5s --start-period=20s --retries=3 \
|
| 39 |
+
CMD curl -sf http://localhost:8000/health || exit 1
|
| 40 |
+
|
| 41 |
+
CMD ["sh", "-c", "cd /app/env && uvicorn server.app:app --host 0.0.0.0 --port 8000"]
|
README.md
CHANGED
|
@@ -1,10 +1,83 @@
|
|
| 1 |
---
|
| 2 |
-
title: Spreadsheet
|
| 3 |
-
emoji:
|
| 4 |
-
colorFrom:
|
| 5 |
-
colorTo:
|
| 6 |
sdk: docker
|
| 7 |
pinned: false
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 8 |
---
|
| 9 |
|
| 10 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
---
|
| 2 |
+
title: "Spreadsheet Environment Server"
|
| 3 |
+
emoji: 📊
|
| 4 |
+
colorFrom: green
|
| 5 |
+
colorTo: blue
|
| 6 |
sdk: docker
|
| 7 |
pinned: false
|
| 8 |
+
license: mit
|
| 9 |
+
app_port: 8000
|
| 10 |
+
base_path: /web
|
| 11 |
+
tags:
|
| 12 |
+
- openenv
|
| 13 |
+
- rl-environment
|
| 14 |
---
|
| 15 |
|
| 16 |
+
# Spreadsheet Environment
|
| 17 |
+
|
| 18 |
+
Exact workbook manipulation and reasoning over realistic spreadsheet tasks. This gym targets weaknesses in structured state tracking, cross-sheet reasoning, non-standard table layouts, and exact edit correctness.
|
| 19 |
+
|
| 20 |
+
## Quick Start
|
| 21 |
+
|
| 22 |
+
```bash
|
| 23 |
+
cd spreadsheet && docker build -t openenv-spreadsheet -f server/Dockerfile .
|
| 24 |
+
docker run -d --name spreadsheet -p 8000:8000 openenv-spreadsheet
|
| 25 |
+
curl http://localhost:8000/health
|
| 26 |
+
```
|
| 27 |
+
|
| 28 |
+
```python
|
| 29 |
+
from spreadsheet import SpreadsheetEnv
|
| 30 |
+
|
| 31 |
+
with SpreadsheetEnv(base_url="http://localhost:8000") as env:
|
| 32 |
+
result = env.reset()
|
| 33 |
+
# Use MCP tools: list_sheets, read_range, write_cell, submit_workbook, etc.
|
| 34 |
+
```
|
| 35 |
+
|
| 36 |
+
## Project Structure
|
| 37 |
+
|
| 38 |
+
```
|
| 39 |
+
spreadsheet/
|
| 40 |
+
├── __init__.py
|
| 41 |
+
├── client.py
|
| 42 |
+
├── models.py
|
| 43 |
+
├── openenv.yaml
|
| 44 |
+
├── pyproject.toml
|
| 45 |
+
├── README.md
|
| 46 |
+
├── .env
|
| 47 |
+
├── .dockerignore
|
| 48 |
+
├── uv.lock
|
| 49 |
+
├── server/
|
| 50 |
+
│ ├── __init__.py
|
| 51 |
+
│ ├── app.py
|
| 52 |
+
│ ├── spreadsheet_environment.py
|
| 53 |
+
│ ├── workbook_engine.py
|
| 54 |
+
│ ├── formula_utils.py
|
| 55 |
+
│ └── scenario_loader.py
|
| 56 |
+
├── workbooks/
|
| 57 |
+
│ ├── templates/
|
| 58 |
+
│ ├── fixtures/
|
| 59 |
+
│ └── hidden_tests/
|
| 60 |
+
├── scenarios/
|
| 61 |
+
└── server/Dockerfile
|
| 62 |
+
```
|
| 63 |
+
|
| 64 |
+
## Reward System
|
| 65 |
+
|
| 66 |
+
Both reward modes use a unified scoring formula:
|
| 67 |
+
|
| 68 |
+
```
|
| 69 |
+
total = 0.25 × quality + 0.15 × efficiency + 0.60 × ground_truth + penalty
|
| 70 |
+
```
|
| 71 |
+
|
| 72 |
+
- **Quality (0.25)** — Custom mode: F1 of expected vs used tools + success rate. OpenEnv mode: fraction of non-neutral steps that were productive (sign-based).
|
| 73 |
+
- **Efficiency (0.15)** — `1.0 - (actual_steps / max_steps)`. Fewer steps = higher score.
|
| 74 |
+
- **Ground Truth (0.60)** — Outcome checks verified against submit_workbook hidden test results (pass rate of cell/formula checks).
|
| 75 |
+
- **Penalty** — Graduated: -0.5 (all calls succeed, 0% ground truth) or -0.2 (<30% ground truth).
|
| 76 |
+
|
| 77 |
+
See [Reward System](../docs/reward-system.md) for full details.
|
| 78 |
+
|
| 79 |
+
## Deployment
|
| 80 |
+
|
| 81 |
+
```bash
|
| 82 |
+
openenv push . --private --repo-id huzzle-labs/spreadsheet
|
| 83 |
+
```
|
__init__.py
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Spreadsheet Environment."""
|
| 2 |
+
|
| 3 |
+
from .client import SpreadsheetEnv
|
| 4 |
+
from .models import SpreadsheetAction, SpreadsheetObservation, SpreadsheetState
|
| 5 |
+
|
| 6 |
+
__all__ = [
|
| 7 |
+
"SpreadsheetAction",
|
| 8 |
+
"SpreadsheetObservation",
|
| 9 |
+
"SpreadsheetState",
|
| 10 |
+
"SpreadsheetEnv",
|
| 11 |
+
]
|
client.py
ADDED
|
@@ -0,0 +1,100 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Spreadsheet Environment Client.
|
| 2 |
+
|
| 3 |
+
Connects to a running Spreadsheet OpenEnv server over HTTP/WebSocket.
|
| 4 |
+
Agents interact via MCP tools (read_range, write_cell, submit_workbook, etc.).
|
| 5 |
+
"""
|
| 6 |
+
|
| 7 |
+
from __future__ import annotations
|
| 8 |
+
|
| 9 |
+
from typing import Any, Dict
|
| 10 |
+
|
| 11 |
+
from openenv.core.client_types import StepResult
|
| 12 |
+
from openenv.core.env_client import EnvClient
|
| 13 |
+
from openenv.core.env_server.mcp_types import CallToolAction, ListToolsAction, Tool
|
| 14 |
+
|
| 15 |
+
from .models import SpreadsheetAction, SpreadsheetObservation, SpreadsheetState
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
class SpreadsheetEnv(
|
| 19 |
+
EnvClient[SpreadsheetAction, SpreadsheetObservation, SpreadsheetState]
|
| 20 |
+
):
|
| 21 |
+
"""Client for the Spreadsheet Environment.
|
| 22 |
+
|
| 23 |
+
Example:
|
| 24 |
+
>>> with SpreadsheetEnv(base_url="http://localhost:8000") as client:
|
| 25 |
+
... result = client.reset()
|
| 26 |
+
... result = client.step(
|
| 27 |
+
... SpreadsheetAction(tool_name="list_scenarios", arguments_json="{}")
|
| 28 |
+
... )
|
| 29 |
+
... result = client.step(
|
| 30 |
+
... SpreadsheetAction(
|
| 31 |
+
... tool_name="read_range",
|
| 32 |
+
... arguments_json='{"sheet":"Summary","range":"A1:D10"}'
|
| 33 |
+
... )
|
| 34 |
+
... )
|
| 35 |
+
"""
|
| 36 |
+
|
| 37 |
+
def list_tools(self, use_cache: bool = True):
|
| 38 |
+
if use_cache and hasattr(self, "_tools_cache") and self._tools_cache:
|
| 39 |
+
return self._tools_cache
|
| 40 |
+
import requests
|
| 41 |
+
|
| 42 |
+
http_base = (
|
| 43 |
+
self._ws_url
|
| 44 |
+
.replace("ws://", "http://")
|
| 45 |
+
.replace("wss://", "https://")
|
| 46 |
+
.rstrip("/ws")
|
| 47 |
+
)
|
| 48 |
+
resp = requests.post(
|
| 49 |
+
f"{http_base}/step",
|
| 50 |
+
json={"action": {"type": "list_tools"}},
|
| 51 |
+
)
|
| 52 |
+
data = resp.json()
|
| 53 |
+
raw_tools = data.get("observation", {}).get("tools", [])
|
| 54 |
+
tools = [
|
| 55 |
+
Tool(
|
| 56 |
+
name=t["name"],
|
| 57 |
+
description=t.get("description", ""),
|
| 58 |
+
input_schema=t.get("input_schema", {}),
|
| 59 |
+
)
|
| 60 |
+
for t in raw_tools
|
| 61 |
+
]
|
| 62 |
+
self._tools_cache = tools
|
| 63 |
+
return tools
|
| 64 |
+
|
| 65 |
+
def _step_payload(self, action: Any) -> Dict:
|
| 66 |
+
if hasattr(action, "to_mcp_action"):
|
| 67 |
+
action = action.to_mcp_action()
|
| 68 |
+
if isinstance(action, ListToolsAction):
|
| 69 |
+
return {"type": "list_tools"}
|
| 70 |
+
if isinstance(action, CallToolAction):
|
| 71 |
+
return {
|
| 72 |
+
"type": "call_tool",
|
| 73 |
+
"tool_name": action.tool_name,
|
| 74 |
+
"arguments": action.arguments or {},
|
| 75 |
+
}
|
| 76 |
+
if hasattr(action, "model_dump"):
|
| 77 |
+
return action.model_dump()
|
| 78 |
+
return {"tool_name": getattr(action, "tool_name", ""), "arguments": {}}
|
| 79 |
+
|
| 80 |
+
def _parse_result(self, payload: Dict) -> StepResult[SpreadsheetObservation]:
|
| 81 |
+
obs_data = payload.get("observation", payload)
|
| 82 |
+
observation = SpreadsheetObservation(
|
| 83 |
+
tool_name=obs_data.get("tool_name", ""),
|
| 84 |
+
result=obs_data.get("result"),
|
| 85 |
+
error=obs_data.get("error"),
|
| 86 |
+
done=payload.get("done", False),
|
| 87 |
+
reward=payload.get("reward"),
|
| 88 |
+
metadata=obs_data.get("metadata", {}),
|
| 89 |
+
)
|
| 90 |
+
return StepResult(
|
| 91 |
+
observation=observation,
|
| 92 |
+
reward=payload.get("reward"),
|
| 93 |
+
done=payload.get("done", False),
|
| 94 |
+
)
|
| 95 |
+
|
| 96 |
+
def _parse_state(self, payload: Dict[str, Any]) -> SpreadsheetState:
|
| 97 |
+
return SpreadsheetState(
|
| 98 |
+
episode_id=payload.get("episode_id"),
|
| 99 |
+
step_count=payload.get("step_count", 0),
|
| 100 |
+
)
|
comparison.md
ADDED
|
@@ -0,0 +1,96 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Spreadsheet Gym — Model Comparison
|
| 2 |
+
|
| 3 |
+
**5 SOTA models × 2 reward modes** evaluated on 12 scenarios.
|
| 4 |
+
|
| 5 |
+
## Summary
|
| 6 |
+
|
| 7 |
+
| Model | Custom Avg | OpenEnv Avg | GT Pass Rate | Avg Steps | Time |
|
| 8 |
+
|---|:---:|:---:|:---:|:---:|:---:|
|
| 9 |
+
| **gpt-5.4** | **0.67** | **0.65** | 10/12 (83%) | 7.8 | 161s |
|
| 10 |
+
| claude-opus-4-20250514 | 0.39 | 0.46 | 7/12 (58%) | 33.6 | 1759s |
|
| 11 |
+
| claude-sonnet-4-6 | 0.33 | 0.44 | 6/12 (50%) | 18.9 | 895s |
|
| 12 |
+
| claude-opus-4-6 | -0.03 | 0.39 | 5/12 (42%) | 41.3 | 1062s |
|
| 13 |
+
| gpt-5 | -0.44 | 0.14 | 1/12 (8%) | 8.8 | 1876s |
|
| 14 |
+
|
| 15 |
+
**Best model:** gpt-5.4 — highest scores on both reward modes, fastest execution, most scenarios solved.
|
| 16 |
+
|
| 17 |
+
## Per-Scenario Breakdown (Custom Mode)
|
| 18 |
+
|
| 19 |
+
| Scenario | gpt-5.4 | sonnet-4-6 | opus-4-6 | opus-0514 | gpt-5 |
|
| 20 |
+
|---|:---:|:---:|:---:|:---:|:---:|
|
| 21 |
+
| buggy_template_fix_01 | **0.92** | 0.94 | 0.94 | 0.85 | 0.89 |
|
| 22 |
+
| conditional_aggregation_01 | **0.96** | 0.87 | 0.81 | -0.78 | -0.71 |
|
| 23 |
+
| conditional_aggregation_02 | **0.91** | -0.68 | -0.66 | 0.85 | -0.68 |
|
| 24 |
+
| cross_sheet_lookup_01 | **0.92** | -0.68 | -0.70 | -0.80 | -0.68 |
|
| 25 |
+
| cross_sheet_lookup_02 | **0.95** | 0.23 | 0.82 | 0.16 | -0.68 |
|
| 26 |
+
| formula_repair_01 | **0.88** | 0.88 | -0.69 | 0.86 | -0.72 |
|
| 27 |
+
| formula_repair_02 | **0.93** | 0.89 | 0.91 | 0.91 | 0.92 |
|
| 28 |
+
| ledger_reconciliation_01 | -0.66 | 0.28 | -0.68 | **0.83** | -0.68 |
|
| 29 |
+
| ledger_reconciliation_02 | **0.92** | 0.22 | -0.66 | 0.13 | -0.80 |
|
| 30 |
+
| messy_table_extraction_01 | -0.66 | **0.83** | -0.68 | ERROR | -0.75 |
|
| 31 |
+
| range_transformation_01 | **0.98** | 0.86 | 0.91 | 0.83 | -0.68 |
|
| 32 |
+
| schedule_grid_fill_01 | **0.96** | -0.68 | -0.68 | 0.88 | -0.69 |
|
| 33 |
+
|
| 34 |
+
## Per-Scenario Breakdown (OpenEnv Mode)
|
| 35 |
+
|
| 36 |
+
| Scenario | gpt-5.4 | sonnet-4-6 | opus-4-6 | opus-0514 | gpt-5 |
|
| 37 |
+
|---|:---:|:---:|:---:|:---:|:---:|
|
| 38 |
+
| buggy_template_fix_01 | **0.76** | 0.76 | 0.76 | 0.74 | 0.15 |
|
| 39 |
+
| conditional_aggregation_01 | **0.76** | 0.74 | 0.73 | 0.74 | 0.14 |
|
| 40 |
+
| conditional_aggregation_02 | **0.76** | 0.14 | 0.14 | 0.74 | 0.14 |
|
| 41 |
+
| cross_sheet_lookup_01 | **0.75** | 0.14 | 0.14 | 0.14 | 0.14 |
|
| 42 |
+
| cross_sheet_lookup_02 | **0.76** | 0.74 | 0.73 | 0.74 | 0.14 |
|
| 43 |
+
| formula_repair_01 | **0.75** | 0.14 | 0.14 | ERROR | 0.15 |
|
| 44 |
+
| formula_repair_02 | **0.76** | 0.75 | 0.75 | ERROR | 0.15 |
|
| 45 |
+
| ledger_reconciliation_01 | 0.14 | **0.74** | 0.14 | ERROR | 0.14 |
|
| 46 |
+
| ledger_reconciliation_02 | **0.75** | 0.14 | 0.14 | 0.13 | 0.14 |
|
| 47 |
+
| messy_table_extraction_01 | 0.14 | 0.13 | 0.14 | **0.75** | 0.14 |
|
| 48 |
+
| range_transformation_01 | **0.76** | 0.76 | 0.75 | 0.74 | 0.14 |
|
| 49 |
+
| schedule_grid_fill_01 | **0.76** | 0.14 | 0.14 | 0.75 | 0.14 |
|
| 50 |
+
|
| 51 |
+
## Step Count Comparison
|
| 52 |
+
|
| 53 |
+
| Scenario | max_steps | gpt-5.4 | sonnet-4-6 | opus-4-6 | opus-0514 | gpt-5 |
|
| 54 |
+
|---|:---:|:---:|:---:|:---:|:---:|:---:|
|
| 55 |
+
| buggy_template_fix_01 | 50 | **10** | 9 | 10 | 26 | 15 |
|
| 56 |
+
| conditional_aggregation_01 | 55 | **6** | 10 | 234 | 55 | 9 |
|
| 57 |
+
| conditional_aggregation_02 | 55 | **8** | 5 | 4 | 12 | 5 |
|
| 58 |
+
| cross_sheet_lookup_01 | 60 | **9** | 6 | 7 | 60 | 5 |
|
| 59 |
+
| cross_sheet_lookup_02 | 60 | **7** | 84 | 198 | 60 | 6 |
|
| 60 |
+
| formula_repair_01 | 40 | **12** | 15 | 7 | 27 | 12 |
|
| 61 |
+
| formula_repair_02 | 40 | **8** | 15 | 13 | 13 | 11 |
|
| 62 |
+
| ledger_reconciliation_01 | 60 | 3 | 7 | 5 | **23** | 5 |
|
| 63 |
+
| ledger_reconciliation_02 | 60 | **9** | 18 | 5 | 55 | 21 |
|
| 64 |
+
| messy_table_extraction_01 | 55 | 3 | **15** | 4 | 17 | 8 |
|
| 65 |
+
| range_transformation_01 | 50 | **5** | 12 | 7 | 39 | 4 |
|
| 66 |
+
| schedule_grid_fill_01 | 55 | **8** | 5 | 5 | 26 | 6 |
|
| 67 |
+
|
| 68 |
+
Bold = model that solved the scenario (GT=1.0) in fewest steps.
|
| 69 |
+
|
| 70 |
+
## Key Observations
|
| 71 |
+
|
| 72 |
+
### 1. Step Score Compression (OpenEnv Mode)
|
| 73 |
+
|
| 74 |
+
All step_score values fall between 0.31–0.41 regardless of agent quality. This is a known issue — the per-step reward magnitudes are too small (0.02–0.50) relative to the normalizer's expected range [-0.5, +1.0]. The entire ranking comes from ground truth alone.
|
| 75 |
+
|
| 76 |
+
### 2. Hallucination Penalty Dominance (Custom Mode)
|
| 77 |
+
|
| 78 |
+
The -1.0 hallucination penalty fires frequently (when all tool calls succeed but ground truth fails). This causes scores like -0.66 to -0.80, making the average for models that fail a few scenarios deeply negative — even if they perform well on others.
|
| 79 |
+
|
| 80 |
+
### 3. Efficiency Score Bug (Custom Mode)
|
| 81 |
+
|
| 82 |
+
The efficiency denominator uses `len(expected_tools)` (tool types, not steps). A scenario with 8 expected tool types but a realistic 20-step solution gives efficiency = 8/20 = 0.40 even for a perfect agent.
|
| 83 |
+
|
| 84 |
+
### 4. gpt-5 OpenEnv Anomaly
|
| 85 |
+
|
| 86 |
+
gpt-5 scored 0.14 across ALL scenarios in OpenEnv mode (GT=0.0 for every scenario). The same model scored higher on some scenarios in custom mode. This suggests a session or environment issue during the OpenEnv batch run, not a model capability problem.
|
| 87 |
+
|
| 88 |
+
### 5. Step Count vs Quality
|
| 89 |
+
|
| 90 |
+
Some models take extremely many steps (opus-4-6: 234 on conditional_aggregation_01, 198 on cross_sheet_lookup_02) but still achieve GT=1.0. Others fail in just 3-5 steps. The current scoring doesn't properly differentiate efficient correct agents from slow correct agents.
|
| 91 |
+
|
| 92 |
+
## Hardening Assessment
|
| 93 |
+
|
| 94 |
+
**SOTA average (custom): 0.67** (gpt-5.4) — below the 0.7 threshold.
|
| 95 |
+
|
| 96 |
+
No hardening required at this time. The scenarios are challenging enough — even the best model fails 2/12 scenarios.
|
generate_scenarios.py
ADDED
|
@@ -0,0 +1,1336 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Generate all 12 scenario workbooks, scenario JSONs, and hidden test JSONs.
|
| 2 |
+
|
| 3 |
+
Run once: cd spreadsheet && python generate_scenarios.py
|
| 4 |
+
"""
|
| 5 |
+
|
| 6 |
+
from __future__ import annotations
|
| 7 |
+
|
| 8 |
+
import json
|
| 9 |
+
import os
|
| 10 |
+
import random
|
| 11 |
+
from datetime import date, datetime, timedelta
|
| 12 |
+
from pathlib import Path
|
| 13 |
+
|
| 14 |
+
import openpyxl
|
| 15 |
+
from openpyxl.styles import Font, Alignment, PatternFill
|
| 16 |
+
from openpyxl.utils import get_column_letter
|
| 17 |
+
|
| 18 |
+
random.seed(42)
|
| 19 |
+
|
| 20 |
+
BASE = Path(__file__).resolve().parent
|
| 21 |
+
TEMPLATES_DIR = BASE / "workbooks" / "templates"
|
| 22 |
+
HIDDEN_TESTS_DIR = BASE / "workbooks" / "hidden_tests"
|
| 23 |
+
SCENARIOS_DIR = BASE / "scenarios"
|
| 24 |
+
|
| 25 |
+
TEMPLATES_DIR.mkdir(parents=True, exist_ok=True)
|
| 26 |
+
HIDDEN_TESTS_DIR.mkdir(parents=True, exist_ok=True)
|
| 27 |
+
SCENARIOS_DIR.mkdir(parents=True, exist_ok=True)
|
| 28 |
+
|
| 29 |
+
NAMES = [
|
| 30 |
+
"Alice Chen", "Bob Martinez", "Carol Singh", "David Kim", "Elena Petrov",
|
| 31 |
+
"Frank Okafor", "Grace Yamamoto", "Hector Reyes", "Irene Muller", "James Adebayo",
|
| 32 |
+
"Karen Novak", "Liam O'Brien", "Maria Santos", "Nathan Park", "Olivia Johansson",
|
| 33 |
+
"Peter Andersen", "Quinn Zhao", "Rosa Fernandez", "Samuel Osei", "Tanya Volkov",
|
| 34 |
+
"Uma Krishnan", "Viktor Sokolov", "Wendy Liu", "Xavier Torres", "Yuki Tanaka",
|
| 35 |
+
]
|
| 36 |
+
|
| 37 |
+
PRODUCTS = [
|
| 38 |
+
("PRD-001", "Widget Alpha", "Hardware"),
|
| 39 |
+
("PRD-002", "Widget Beta", "Hardware"),
|
| 40 |
+
("PRD-003", "Service Gold", "Services"),
|
| 41 |
+
("PRD-004", "Service Silver", "Services"),
|
| 42 |
+
("PRD-005", "License Pro", "Software"),
|
| 43 |
+
("PRD-006", "License Basic", "Software"),
|
| 44 |
+
("PRD-007", "Widget Gamma", "Hardware"),
|
| 45 |
+
("PRD-008", "Consulting Plus", "Services"),
|
| 46 |
+
]
|
| 47 |
+
|
| 48 |
+
REGIONS = ["North", "South", "East", "West"]
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
def _write_json(path: Path, data: dict):
|
| 52 |
+
with open(path, "w") as f:
|
| 53 |
+
json.dump(data, f, indent=2)
|
| 54 |
+
|
| 55 |
+
|
| 56 |
+
def _bold_font():
|
| 57 |
+
return Font(bold=True)
|
| 58 |
+
|
| 59 |
+
|
| 60 |
+
def _header_fill():
|
| 61 |
+
return PatternFill(start_color="4472C4", end_color="4472C4", fill_type="solid")
|
| 62 |
+
|
| 63 |
+
|
| 64 |
+
def _header_font():
|
| 65 |
+
return Font(bold=True, color="FFFFFF")
|
| 66 |
+
|
| 67 |
+
|
| 68 |
+
def _yellow_fill():
|
| 69 |
+
return PatternFill(start_color="FFFF00", end_color="FFFF00", fill_type="solid")
|
| 70 |
+
|
| 71 |
+
|
| 72 |
+
# ══════════════════════════════════════════════════════════════════════
|
| 73 |
+
# Scenario 1: formula_repair_01 — Multi-Department Budget Repair
|
| 74 |
+
# ══════════════════════════════════════════════════════════════════════
|
| 75 |
+
|
| 76 |
+
def gen_formula_repair_01():
|
| 77 |
+
wb = openpyxl.Workbook()
|
| 78 |
+
|
| 79 |
+
# -- Engineering sheet --
|
| 80 |
+
ws_eng = wb.active
|
| 81 |
+
ws_eng.title = "Engineering"
|
| 82 |
+
headers = ["Name", "Role", "Base Salary", "Bonus %", "Total Comp"]
|
| 83 |
+
for c, h in enumerate(headers, 1):
|
| 84 |
+
cell = ws_eng.cell(row=1, column=c, value=h)
|
| 85 |
+
cell.font = _header_font()
|
| 86 |
+
cell.fill = _header_fill()
|
| 87 |
+
|
| 88 |
+
roles = ["Senior Engineer", "Staff Engineer", "Principal", "Junior Engineer", "Tech Lead"]
|
| 89 |
+
for i in range(20):
|
| 90 |
+
r = i + 2
|
| 91 |
+
ws_eng.cell(row=r, column=1, value=NAMES[i % len(NAMES)])
|
| 92 |
+
ws_eng.cell(row=r, column=2, value=roles[i % len(roles)])
|
| 93 |
+
salary = random.randint(80, 200) * 1000
|
| 94 |
+
bonus_pct = random.choice([0.05, 0.10, 0.15, 0.20])
|
| 95 |
+
ws_eng.cell(row=r, column=3, value=salary)
|
| 96 |
+
ws_eng.cell(row=r, column=4, value=bonus_pct)
|
| 97 |
+
ws_eng.cell(row=r, column=5, value=f"=C{r}*(1+D{r})")
|
| 98 |
+
# Add a blank separator row
|
| 99 |
+
ws_eng.cell(row=12, column=1, value=None)
|
| 100 |
+
|
| 101 |
+
# -- Marketing sheet --
|
| 102 |
+
ws_mkt = wb.create_sheet("Marketing")
|
| 103 |
+
mkt_headers = ["Name", "Role", "Base Salary", "Campaign Budget", "Total Comp"]
|
| 104 |
+
for c, h in enumerate(mkt_headers, 1):
|
| 105 |
+
cell = ws_mkt.cell(row=1, column=c, value=h)
|
| 106 |
+
cell.font = _header_font()
|
| 107 |
+
cell.fill = _header_fill()
|
| 108 |
+
|
| 109 |
+
mkt_roles = ["Marketing Manager", "Content Lead", "SEO Specialist", "Brand Director"]
|
| 110 |
+
for i in range(15):
|
| 111 |
+
r = i + 2
|
| 112 |
+
ws_mkt.cell(row=r, column=1, value=NAMES[(i + 5) % len(NAMES)])
|
| 113 |
+
ws_mkt.cell(row=r, column=2, value=mkt_roles[i % len(mkt_roles)])
|
| 114 |
+
salary = random.randint(60, 150) * 1000
|
| 115 |
+
ws_mkt.cell(row=r, column=3, value=salary)
|
| 116 |
+
ws_mkt.cell(row=r, column=4, value=random.randint(5, 50) * 1000)
|
| 117 |
+
# BUG: References a deleted "OldBudget" sheet
|
| 118 |
+
ws_mkt.cell(row=r, column=5, value=f"=C{r}+OldBudget!B{r}")
|
| 119 |
+
|
| 120 |
+
# -- HR Policies sheet (non-standard layout) --
|
| 121 |
+
ws_hr = wb.create_sheet("HR Policies")
|
| 122 |
+
ws_hr.cell(row=1, column=1, value="Company Bonus Policy").font = _bold_font()
|
| 123 |
+
policies = [
|
| 124 |
+
"All employees are eligible for annual performance bonuses.",
|
| 125 |
+
"Bonus tiers: Junior 5%, Mid 10%, Senior 15%, Principal 20%.",
|
| 126 |
+
"Bonuses are calculated on base salary before taxes.",
|
| 127 |
+
"Campaign budgets are separate from compensation.",
|
| 128 |
+
"Total compensation = Base Salary × (1 + Bonus %).",
|
| 129 |
+
]
|
| 130 |
+
for i, p in enumerate(policies):
|
| 131 |
+
ws_hr.cell(row=i + 3, column=1, value=p)
|
| 132 |
+
|
| 133 |
+
# Lookup table starts at F1 (non-standard)
|
| 134 |
+
ws_hr.cell(row=1, column=6, value="Tier").font = _bold_font()
|
| 135 |
+
ws_hr.cell(row=1, column=7, value="Min Bonus %").font = _bold_font()
|
| 136 |
+
ws_hr.cell(row=1, column=8, value="Max Bonus %").font = _bold_font()
|
| 137 |
+
tiers = [("Junior", 0.05, 0.08), ("Mid", 0.10, 0.12), ("Senior", 0.15, 0.18), ("Principal", 0.20, 0.25)]
|
| 138 |
+
for i, (tier, mn, mx) in enumerate(tiers):
|
| 139 |
+
ws_hr.cell(row=i + 2, column=6, value=tier)
|
| 140 |
+
ws_hr.cell(row=i + 2, column=7, value=mn)
|
| 141 |
+
ws_hr.cell(row=i + 2, column=8, value=mx)
|
| 142 |
+
|
| 143 |
+
# -- Summary sheet --
|
| 144 |
+
ws_sum = wb.create_sheet("Summary")
|
| 145 |
+
ws_sum.cell(row=1, column=1, value="Department Budget Summary").font = Font(bold=True, size=14)
|
| 146 |
+
ws_sum.merge_cells("A1:C1")
|
| 147 |
+
|
| 148 |
+
ws_sum.cell(row=3, column=1, value="Department").font = _bold_font()
|
| 149 |
+
ws_sum.cell(row=3, column=2, value="Total Base Salary").font = _bold_font()
|
| 150 |
+
ws_sum.cell(row=3, column=3, value="Total Compensation").font = _bold_font()
|
| 151 |
+
|
| 152 |
+
ws_sum.cell(row=4, column=1, value="Engineering")
|
| 153 |
+
ws_sum.cell(row=4, column=2, value="=SUM(Engineering!C2:C21)")
|
| 154 |
+
# BUG: wrong range (C2:C10 instead of E2:E21) and references wrong sheet
|
| 155 |
+
ws_sum.cell(row=4, column=3, value="=SUM(Engineering!C2:C10)")
|
| 156 |
+
|
| 157 |
+
ws_sum.cell(row=5, column=1, value="Marketing")
|
| 158 |
+
ws_sum.cell(row=5, column=2, value="=SUM(Marketing!C2:C16)")
|
| 159 |
+
# BUG: references deleted OldBudget sheet
|
| 160 |
+
ws_sum.cell(row=5, column=3, value="=SUM(OldBudget!E2:E16)")
|
| 161 |
+
|
| 162 |
+
ws_sum.cell(row=7, column=1, value="Grand Total").font = _bold_font()
|
| 163 |
+
ws_sum.cell(row=7, column=2, value="=B4+B5")
|
| 164 |
+
# BUG: formula should sum total comp, not base salary again
|
| 165 |
+
ws_sum.cell(row=7, column=3, value="=B4+B5")
|
| 166 |
+
|
| 167 |
+
# -- Metadata sheet (hidden) --
|
| 168 |
+
ws_meta = wb.create_sheet("Metadata")
|
| 169 |
+
ws_meta.sheet_state = "hidden"
|
| 170 |
+
ws_meta.cell(row=1, column=1, value="Target: Fix Summary!C4, Summary!C5, Summary!C7")
|
| 171 |
+
ws_meta.cell(row=2, column=1, value="Target: Fix Marketing total comp formulas (remove OldBudget refs)")
|
| 172 |
+
ws_meta.cell(row=3, column=1, value="Expected: C7 = sum of all total comp across both departments")
|
| 173 |
+
|
| 174 |
+
wb.move_sheet("Summary", offset=-3)
|
| 175 |
+
wb.save(TEMPLATES_DIR / "multi_department_budget.xlsx")
|
| 176 |
+
|
| 177 |
+
# Scenario JSON
|
| 178 |
+
_write_json(SCENARIOS_DIR / "formula_repair_01.json", {
|
| 179 |
+
"id": "formula_repair_01",
|
| 180 |
+
"description": "Fix broken formulas in a multi-department budget workbook. Summary sheet has wrong ranges and references to a deleted sheet. Marketing total comp formulas reference a non-existent OldBudget sheet.",
|
| 181 |
+
"instructions": "The Summary sheet has broken formulas. Engineering total compensation references wrong cell ranges. Marketing total compensation references a deleted 'OldBudget' sheet. Fix all broken formulas so Summary correctly aggregates total compensation from Engineering and Marketing sheets. Also fix the Marketing sheet's Total Comp column to use the correct formula (Base Salary × (1 + Bonus %)). Check the HR Policies sheet for the correct bonus calculation method. There is a hidden Metadata sheet with hints.",
|
| 182 |
+
"workbook": "multi_department_budget.xlsx",
|
| 183 |
+
"max_steps": 50,
|
| 184 |
+
"category": "formula_repair",
|
| 185 |
+
})
|
| 186 |
+
|
| 187 |
+
_write_json(HIDDEN_TESTS_DIR / "formula_repair_01.json", {
|
| 188 |
+
"scenario_id": "formula_repair_01",
|
| 189 |
+
"checks": [
|
| 190 |
+
{"sheet": "Summary", "cell": "C4", "expected_formula": "=SUM(Engineering!E2:E21)"},
|
| 191 |
+
{"sheet": "Summary", "cell": "C5", "expected_formula": "=SUM(Marketing!E2:E16)"},
|
| 192 |
+
{"sheet": "Summary", "cell": "C7", "expected_formula": "=C4+C5"},
|
| 193 |
+
{"sheet": "Marketing", "range": "E2:E16", "check": "no_blanks"},
|
| 194 |
+
{"sheet": "Engineering", "range": "E2:E21", "check": "no_blanks"},
|
| 195 |
+
],
|
| 196 |
+
"target_regions": [
|
| 197 |
+
{"sheet": "Summary", "range": "C4:C7"},
|
| 198 |
+
{"sheet": "Marketing", "range": "E2:E16"},
|
| 199 |
+
],
|
| 200 |
+
})
|
| 201 |
+
|
| 202 |
+
|
| 203 |
+
# ══════════════════════════════════════════════════════════════════════
|
| 204 |
+
# Scenario 2: formula_repair_02 — Cascading Formula Errors
|
| 205 |
+
# ══════════════════════════════════════════════════════════════════════
|
| 206 |
+
|
| 207 |
+
def gen_formula_repair_02():
|
| 208 |
+
wb = openpyxl.Workbook()
|
| 209 |
+
|
| 210 |
+
# -- Assumptions sheet --
|
| 211 |
+
ws_a = wb.active
|
| 212 |
+
ws_a.title = "Assumptions"
|
| 213 |
+
ws_a.cell(row=1, column=1, value="Parameter").font = _bold_font()
|
| 214 |
+
ws_a.cell(row=1, column=2, value="Value").font = _bold_font()
|
| 215 |
+
params = [
|
| 216 |
+
("Growth Rate", 0.08), ("Discount Rate", 0.12), ("Tax Rate", 0.21),
|
| 217 |
+
("Inflation", 0.03), ("Depreciation Years", 5),
|
| 218 |
+
]
|
| 219 |
+
for i, (name, val) in enumerate(params):
|
| 220 |
+
ws_a.cell(row=i + 2, column=1, value=name)
|
| 221 |
+
ws_a.cell(row=i + 2, column=2, value=val)
|
| 222 |
+
|
| 223 |
+
# -- Revenue Projections sheet --
|
| 224 |
+
ws_rev = wb.create_sheet("Revenue")
|
| 225 |
+
ws_rev.cell(row=1, column=1, value="Year").font = _bold_font()
|
| 226 |
+
for y in range(5):
|
| 227 |
+
ws_rev.cell(row=1, column=y + 2, value=2024 + y).font = _bold_font()
|
| 228 |
+
|
| 229 |
+
ws_rev.cell(row=2, column=1, value="Base Revenue")
|
| 230 |
+
ws_rev.cell(row=2, column=2, value=1000000)
|
| 231 |
+
for y in range(1, 5):
|
| 232 |
+
col = y + 2
|
| 233 |
+
prev_col = get_column_letter(col - 1)
|
| 234 |
+
# BUG: References Assumptions!B2 but should use absolute ref, and year 3+ has wrong formula
|
| 235 |
+
if y < 3:
|
| 236 |
+
ws_rev.cell(row=2, column=col, value=f"={prev_col}2*(1+Assumptions!B2)")
|
| 237 |
+
else:
|
| 238 |
+
# BUG: hardcoded 0.05 instead of referencing Assumptions
|
| 239 |
+
ws_rev.cell(row=2, column=col, value=f"={prev_col}2*1.05")
|
| 240 |
+
|
| 241 |
+
ws_rev.cell(row=3, column=1, value="Operating Costs")
|
| 242 |
+
for y in range(5):
|
| 243 |
+
col = y + 2
|
| 244 |
+
ws_rev.cell(row=3, column=col, value=f"={get_column_letter(col)}2*0.65")
|
| 245 |
+
|
| 246 |
+
ws_rev.cell(row=4, column=1, value="EBIT")
|
| 247 |
+
for y in range(5):
|
| 248 |
+
col = y + 2
|
| 249 |
+
cl = get_column_letter(col)
|
| 250 |
+
ws_rev.cell(row=4, column=col, value=f"={cl}2-{cl}3")
|
| 251 |
+
|
| 252 |
+
ws_rev.cell(row=5, column=1, value="Tax")
|
| 253 |
+
for y in range(5):
|
| 254 |
+
col = y + 2
|
| 255 |
+
cl = get_column_letter(col)
|
| 256 |
+
# BUG: Uses hardcoded 0.25 instead of Assumptions!B4 (Tax Rate)
|
| 257 |
+
ws_rev.cell(row=5, column=col, value=f"={cl}4*0.25")
|
| 258 |
+
|
| 259 |
+
ws_rev.cell(row=6, column=1, value="Net Income")
|
| 260 |
+
for y in range(5):
|
| 261 |
+
col = y + 2
|
| 262 |
+
cl = get_column_letter(col)
|
| 263 |
+
ws_rev.cell(row=6, column=col, value=f"={cl}4-{cl}5")
|
| 264 |
+
|
| 265 |
+
# -- DCF sheet --
|
| 266 |
+
ws_dcf = wb.create_sheet("DCF")
|
| 267 |
+
ws_dcf.cell(row=1, column=1, value="Year").font = _bold_font()
|
| 268 |
+
ws_dcf.cell(row=1, column=2, value="Net Income").font = _bold_font()
|
| 269 |
+
ws_dcf.cell(row=1, column=3, value="Discount Factor").font = _bold_font()
|
| 270 |
+
ws_dcf.cell(row=1, column=4, value="PV").font = _bold_font()
|
| 271 |
+
|
| 272 |
+
for y in range(5):
|
| 273 |
+
r = y + 2
|
| 274 |
+
col_letter = get_column_letter(y + 2)
|
| 275 |
+
ws_dcf.cell(row=r, column=1, value=2024 + y)
|
| 276 |
+
ws_dcf.cell(row=r, column=2, value=f"=Revenue!{col_letter}6")
|
| 277 |
+
# BUG: discount factor uses wrong cell ref (B3 = Discount Rate is actually B3 in Assumptions)
|
| 278 |
+
ws_dcf.cell(row=r, column=3, value=f"=1/(1+Assumptions!B3)^{y + 1}")
|
| 279 |
+
ws_dcf.cell(row=r, column=4, value=f"=B{r}*C{r}")
|
| 280 |
+
|
| 281 |
+
ws_dcf.cell(row=8, column=1, value="Total NPV").font = _bold_font()
|
| 282 |
+
ws_dcf.cell(row=8, column=4, value="=SUM(D2:D6)")
|
| 283 |
+
|
| 284 |
+
wb.save(TEMPLATES_DIR / "cascading_formula_errors.xlsx")
|
| 285 |
+
|
| 286 |
+
_write_json(SCENARIOS_DIR / "formula_repair_02.json", {
|
| 287 |
+
"id": "formula_repair_02",
|
| 288 |
+
"description": "Fix cascading formula errors in a 5-year financial projection. Revenue growth, tax rates, and discount factors reference wrong cells or use hardcoded values instead of the Assumptions sheet.",
|
| 289 |
+
"instructions": "This workbook has a 5-year financial projection with three sheets: Assumptions, Revenue, and DCF. Multiple formulas contain errors: (1) Revenue years 4-5 use hardcoded 5% growth instead of the Assumptions growth rate. (2) Tax calculations use 25% instead of the Assumptions tax rate (21%). (3) The Assumptions sheet has the correct values — all formulas should reference it. Fix all broken formulas in Revenue and DCF sheets to properly reference Assumptions.",
|
| 290 |
+
"workbook": "cascading_formula_errors.xlsx",
|
| 291 |
+
"max_steps": 50,
|
| 292 |
+
"category": "formula_repair",
|
| 293 |
+
})
|
| 294 |
+
|
| 295 |
+
_write_json(HIDDEN_TESTS_DIR / "formula_repair_02.json", {
|
| 296 |
+
"scenario_id": "formula_repair_02",
|
| 297 |
+
"checks": [
|
| 298 |
+
{"sheet": "Revenue", "cell": "E2", "expected_formula": "=D2*(1+Assumptions!B2)"},
|
| 299 |
+
{"sheet": "Revenue", "cell": "F2", "expected_formula": "=E2*(1+Assumptions!B2)"},
|
| 300 |
+
{"sheet": "Revenue", "cell": "B5", "expected_formula": "=B4*Assumptions!B4"},
|
| 301 |
+
{"sheet": "Revenue", "cell": "C5", "expected_formula": "=C4*Assumptions!B4"},
|
| 302 |
+
{"sheet": "Revenue", "cell": "D5", "expected_formula": "=D4*Assumptions!B4"},
|
| 303 |
+
{"sheet": "Revenue", "cell": "E5", "expected_formula": "=E4*Assumptions!B4"},
|
| 304 |
+
{"sheet": "Revenue", "cell": "F5", "expected_formula": "=F4*Assumptions!B4"},
|
| 305 |
+
],
|
| 306 |
+
"target_regions": [
|
| 307 |
+
{"sheet": "Revenue", "range": "B2:F6"},
|
| 308 |
+
{"sheet": "DCF", "range": "B2:D8"},
|
| 309 |
+
],
|
| 310 |
+
})
|
| 311 |
+
|
| 312 |
+
|
| 313 |
+
# ══════════════════════════════════════════════════════════════════════
|
| 314 |
+
# Scenario 3: cross_sheet_lookup_01 — Product Revenue by Region
|
| 315 |
+
# ══════════════════════════════════════════════════════════════════════
|
| 316 |
+
|
| 317 |
+
def gen_cross_sheet_lookup_01():
|
| 318 |
+
wb = openpyxl.Workbook()
|
| 319 |
+
|
| 320 |
+
# -- Products sheet (lookup table) --
|
| 321 |
+
ws_prod = wb.active
|
| 322 |
+
ws_prod.title = "Products"
|
| 323 |
+
for c, h in enumerate(["Code", "Name", "Category", "Unit Price"], 1):
|
| 324 |
+
ws_prod.cell(row=1, column=c, value=h).font = _header_font()
|
| 325 |
+
ws_prod.cell(row=1, column=c).fill = _header_fill()
|
| 326 |
+
for i, (code, name, cat) in enumerate(PRODUCTS):
|
| 327 |
+
r = i + 2
|
| 328 |
+
ws_prod.cell(row=r, column=1, value=code)
|
| 329 |
+
ws_prod.cell(row=r, column=2, value=name)
|
| 330 |
+
ws_prod.cell(row=r, column=3, value=cat)
|
| 331 |
+
ws_prod.cell(row=r, column=4, value=random.randint(50, 500))
|
| 332 |
+
|
| 333 |
+
# -- Sales Q1 sheet (raw data with some bad codes) --
|
| 334 |
+
ws_q1 = wb.create_sheet("Sales_Q1")
|
| 335 |
+
q1_headers = ["Date", "Product Code", "Region", "Quantity", "Revenue"]
|
| 336 |
+
for c, h in enumerate(q1_headers, 1):
|
| 337 |
+
ws_q1.cell(row=1, column=c, value=h).font = _bold_font()
|
| 338 |
+
|
| 339 |
+
q1_rows = 80
|
| 340 |
+
for i in range(q1_rows):
|
| 341 |
+
r = i + 2
|
| 342 |
+
d = date(2024, 1, 1) + timedelta(days=random.randint(0, 89))
|
| 343 |
+
code = PRODUCTS[random.randint(0, len(PRODUCTS) - 1)][0]
|
| 344 |
+
# Introduce some bad codes (typos)
|
| 345 |
+
if i in (15, 32, 55, 71):
|
| 346 |
+
code = code.replace("-", "") # PRD001 instead of PRD-001
|
| 347 |
+
region = random.choice(REGIONS)
|
| 348 |
+
qty = random.randint(1, 50)
|
| 349 |
+
ws_q1.cell(row=r, column=1, value=d)
|
| 350 |
+
ws_q1.cell(row=r, column=2, value=code)
|
| 351 |
+
ws_q1.cell(row=r, column=3, value=region)
|
| 352 |
+
ws_q1.cell(row=r, column=4, value=qty)
|
| 353 |
+
ws_q1.cell(row=r, column=5, value=qty * random.randint(50, 500))
|
| 354 |
+
|
| 355 |
+
# -- Sales Q2 sheet --
|
| 356 |
+
ws_q2 = wb.create_sheet("Sales_Q2")
|
| 357 |
+
for c, h in enumerate(q1_headers, 1):
|
| 358 |
+
ws_q2.cell(row=1, column=c, value=h).font = _bold_font()
|
| 359 |
+
|
| 360 |
+
q2_rows = 90
|
| 361 |
+
for i in range(q2_rows):
|
| 362 |
+
r = i + 2
|
| 363 |
+
d = date(2024, 4, 1) + timedelta(days=random.randint(0, 90))
|
| 364 |
+
code = PRODUCTS[random.randint(0, len(PRODUCTS) - 1)][0]
|
| 365 |
+
if i in (20, 45, 67):
|
| 366 |
+
code = code.lower() # prd-003 instead of PRD-003
|
| 367 |
+
region = random.choice(REGIONS)
|
| 368 |
+
qty = random.randint(1, 50)
|
| 369 |
+
ws_q2.cell(row=r, column=1, value=d)
|
| 370 |
+
ws_q2.cell(row=r, column=2, value=code)
|
| 371 |
+
ws_q2.cell(row=r, column=3, value=region)
|
| 372 |
+
ws_q2.cell(row=r, column=4, value=qty)
|
| 373 |
+
ws_q2.cell(row=r, column=5, value=qty * random.randint(50, 500))
|
| 374 |
+
|
| 375 |
+
# -- Summary sheet (agent must fill) --
|
| 376 |
+
ws_sum = wb.create_sheet("Summary")
|
| 377 |
+
ws_sum.cell(row=1, column=1, value="Revenue Summary by Region and Category").font = Font(bold=True, size=14)
|
| 378 |
+
ws_sum.merge_cells("A1:E1")
|
| 379 |
+
|
| 380 |
+
ws_sum.cell(row=3, column=1, value="Region").font = _bold_font()
|
| 381 |
+
ws_sum.cell(row=3, column=2, value="Hardware").font = _bold_font()
|
| 382 |
+
ws_sum.cell(row=3, column=3, value="Services").font = _bold_font()
|
| 383 |
+
ws_sum.cell(row=3, column=4, value="Software").font = _bold_font()
|
| 384 |
+
ws_sum.cell(row=3, column=5, value="Total").font = _bold_font()
|
| 385 |
+
|
| 386 |
+
for i, region in enumerate(REGIONS):
|
| 387 |
+
r = i + 4
|
| 388 |
+
ws_sum.cell(row=r, column=1, value=region)
|
| 389 |
+
for c in range(2, 6):
|
| 390 |
+
cell = ws_sum.cell(row=r, column=c)
|
| 391 |
+
cell.fill = _yellow_fill()
|
| 392 |
+
|
| 393 |
+
ws_sum.cell(row=8, column=1, value="Grand Total").font = _bold_font()
|
| 394 |
+
for c in range(2, 6):
|
| 395 |
+
ws_sum.cell(row=8, column=c).fill = _yellow_fill()
|
| 396 |
+
|
| 397 |
+
wb.save(TEMPLATES_DIR / "product_revenue_by_region.xlsx")
|
| 398 |
+
|
| 399 |
+
_write_json(SCENARIOS_DIR / "cross_sheet_lookup_01.json", {
|
| 400 |
+
"id": "cross_sheet_lookup_01",
|
| 401 |
+
"description": "Aggregate product revenue by region and category across two quarterly sales sheets. Some product codes have typos. The Summary sheet must be filled with correct totals.",
|
| 402 |
+
"instructions": "Fill the Summary sheet with revenue totals broken down by Region (rows) and Product Category (columns: Hardware, Services, Software). Data is in Sales_Q1 and Sales_Q2. Use the Products sheet to map product codes to categories. WARNING: Some product codes in the sales sheets have typos (missing dashes or lowercase). You must account for these when aggregating. The Total column should sum across categories for each region. Grand Total row should sum each column.",
|
| 403 |
+
"workbook": "product_revenue_by_region.xlsx",
|
| 404 |
+
"max_steps": 60,
|
| 405 |
+
"category": "cross_sheet_lookup",
|
| 406 |
+
})
|
| 407 |
+
|
| 408 |
+
# Calculate expected values
|
| 409 |
+
cat_map = {code: cat for code, _, cat in PRODUCTS}
|
| 410 |
+
# Also map typo variants
|
| 411 |
+
for code, _, cat in PRODUCTS:
|
| 412 |
+
cat_map[code.replace("-", "")] = cat
|
| 413 |
+
cat_map[code.lower()] = cat
|
| 414 |
+
|
| 415 |
+
totals = {r: {"Hardware": 0, "Services": 0, "Software": 0} for r in REGIONS}
|
| 416 |
+
for ws_name in ["Sales_Q1", "Sales_Q2"]:
|
| 417 |
+
ws = wb[ws_name]
|
| 418 |
+
for row in ws.iter_rows(min_row=2, values_only=True):
|
| 419 |
+
_, code, region, _, revenue = row
|
| 420 |
+
if code is None:
|
| 421 |
+
continue
|
| 422 |
+
cat = cat_map.get(str(code))
|
| 423 |
+
if cat and region in totals:
|
| 424 |
+
totals[region][cat] += revenue
|
| 425 |
+
|
| 426 |
+
checks = []
|
| 427 |
+
for i, region in enumerate(REGIONS):
|
| 428 |
+
r = i + 4
|
| 429 |
+
for j, cat in enumerate(["Hardware", "Services", "Software"]):
|
| 430 |
+
col = get_column_letter(j + 2)
|
| 431 |
+
val = totals[region][cat]
|
| 432 |
+
checks.append({
|
| 433 |
+
"sheet": "Summary", "cell": f"{col}{r}",
|
| 434 |
+
"expected_value_range": [val * 0.99, val * 1.01],
|
| 435 |
+
})
|
| 436 |
+
|
| 437 |
+
checks.append({"sheet": "Summary", "range": "B4:E8", "check": "no_blanks"})
|
| 438 |
+
|
| 439 |
+
_write_json(HIDDEN_TESTS_DIR / "cross_sheet_lookup_01.json", {
|
| 440 |
+
"scenario_id": "cross_sheet_lookup_01",
|
| 441 |
+
"checks": checks,
|
| 442 |
+
"target_regions": [{"sheet": "Summary", "range": "B4:E8"}],
|
| 443 |
+
})
|
| 444 |
+
|
| 445 |
+
|
| 446 |
+
# ══════════════════════════════════════════════════════════════════════
|
| 447 |
+
# Scenario 4: cross_sheet_lookup_02 — Employee Bonus Calculation
|
| 448 |
+
# ══════════════════════════════════════════════════════════════════════
|
| 449 |
+
|
| 450 |
+
def gen_cross_sheet_lookup_02():
|
| 451 |
+
wb = openpyxl.Workbook()
|
| 452 |
+
|
| 453 |
+
# -- Employees --
|
| 454 |
+
ws_emp = wb.active
|
| 455 |
+
ws_emp.title = "Employees"
|
| 456 |
+
for c, h in enumerate(["ID", "Name", "Department", "Level", "Base Salary"], 1):
|
| 457 |
+
ws_emp.cell(row=1, column=c, value=h).font = _bold_font()
|
| 458 |
+
|
| 459 |
+
levels = ["Junior", "Mid", "Senior", "Principal"]
|
| 460 |
+
depts = ["Engineering", "Marketing", "Sales", "Operations"]
|
| 461 |
+
for i in range(25):
|
| 462 |
+
r = i + 2
|
| 463 |
+
ws_emp.cell(row=r, column=1, value=f"EMP-{i+1:03d}")
|
| 464 |
+
ws_emp.cell(row=r, column=2, value=NAMES[i])
|
| 465 |
+
ws_emp.cell(row=r, column=3, value=depts[i % len(depts)])
|
| 466 |
+
ws_emp.cell(row=r, column=4, value=levels[i % len(levels)])
|
| 467 |
+
ws_emp.cell(row=r, column=5, value=random.randint(50, 200) * 1000)
|
| 468 |
+
|
| 469 |
+
# -- Bonus Tiers (non-standard layout: starts at column F) --
|
| 470 |
+
ws_tiers = wb.create_sheet("Bonus_Tiers")
|
| 471 |
+
ws_tiers.cell(row=1, column=1, value="This sheet contains the bonus tier lookup table.")
|
| 472 |
+
ws_tiers.cell(row=2, column=1, value="The table is located in columns F-I, not A-D.")
|
| 473 |
+
ws_tiers.cell(row=3, column=1, value="Do not modify columns A-D.")
|
| 474 |
+
|
| 475 |
+
ws_tiers.cell(row=1, column=6, value="Level").font = _bold_font()
|
| 476 |
+
ws_tiers.cell(row=1, column=7, value="Bonus Rate").font = _bold_font()
|
| 477 |
+
ws_tiers.cell(row=1, column=8, value="Min Performance").font = _bold_font()
|
| 478 |
+
ws_tiers.cell(row=1, column=9, value="Cap Multiplier").font = _bold_font()
|
| 479 |
+
tier_data = [
|
| 480 |
+
("Junior", 0.05, 3, 1.0), ("Mid", 0.10, 3, 1.2),
|
| 481 |
+
("Senior", 0.15, 4, 1.5), ("Principal", 0.20, 4, 2.0),
|
| 482 |
+
]
|
| 483 |
+
for i, (level, rate, min_perf, cap) in enumerate(tier_data):
|
| 484 |
+
r = i + 2
|
| 485 |
+
ws_tiers.cell(row=r, column=6, value=level)
|
| 486 |
+
ws_tiers.cell(row=r, column=7, value=rate)
|
| 487 |
+
ws_tiers.cell(row=r, column=8, value=min_perf)
|
| 488 |
+
ws_tiers.cell(row=r, column=9, value=cap)
|
| 489 |
+
|
| 490 |
+
# -- Performance scores --
|
| 491 |
+
ws_perf = wb.create_sheet("Performance")
|
| 492 |
+
for c, h in enumerate(["Employee ID", "Q1", "Q2", "Q3", "Q4", "Avg Score"], 1):
|
| 493 |
+
ws_perf.cell(row=1, column=c, value=h).font = _bold_font()
|
| 494 |
+
for i in range(25):
|
| 495 |
+
r = i + 2
|
| 496 |
+
ws_perf.cell(row=r, column=1, value=f"EMP-{i+1:03d}")
|
| 497 |
+
scores = [random.randint(1, 5) for _ in range(4)]
|
| 498 |
+
for q in range(4):
|
| 499 |
+
ws_perf.cell(row=r, column=q + 2, value=scores[q])
|
| 500 |
+
ws_perf.cell(row=r, column=6, value=f"=AVERAGE(B{r}:E{r})")
|
| 501 |
+
|
| 502 |
+
# -- Payroll (agent must fill) --
|
| 503 |
+
ws_pay = wb.create_sheet("Payroll")
|
| 504 |
+
for c, h in enumerate(["Employee ID", "Name", "Level", "Base Salary", "Avg Score", "Bonus Rate", "Bonus Amount", "Total Comp"], 1):
|
| 505 |
+
cell = ws_pay.cell(row=1, column=c, value=h)
|
| 506 |
+
cell.font = _bold_font()
|
| 507 |
+
for i in range(25):
|
| 508 |
+
r = i + 2
|
| 509 |
+
ws_pay.cell(row=r, column=1, value=f"EMP-{i+1:03d}")
|
| 510 |
+
for c in range(2, 9):
|
| 511 |
+
ws_pay.cell(row=r, column=c).fill = _yellow_fill()
|
| 512 |
+
|
| 513 |
+
wb.save(TEMPLATES_DIR / "employee_bonus_calculation.xlsx")
|
| 514 |
+
|
| 515 |
+
_write_json(SCENARIOS_DIR / "cross_sheet_lookup_02.json", {
|
| 516 |
+
"id": "cross_sheet_lookup_02",
|
| 517 |
+
"description": "Calculate employee bonuses by cross-referencing Employees, Bonus_Tiers (non-standard layout at column F), and Performance sheets. Fill the Payroll sheet.",
|
| 518 |
+
"instructions": "Fill the Payroll sheet for all 25 employees. For each employee: (1) Look up their Name, Level, and Base Salary from the Employees sheet. (2) Look up their average performance score from the Performance sheet. (3) Find the bonus rate from the Bonus_Tiers sheet (NOTE: the tier table is in columns F-I, not A-D). (4) If the employee's avg score meets the minimum performance threshold for their tier, apply the bonus rate; otherwise bonus is 0. (5) Bonus Amount = Base Salary × Bonus Rate. (6) Total Comp = Base Salary + Bonus Amount.",
|
| 519 |
+
"workbook": "employee_bonus_calculation.xlsx",
|
| 520 |
+
"max_steps": 60,
|
| 521 |
+
"category": "cross_sheet_lookup",
|
| 522 |
+
})
|
| 523 |
+
|
| 524 |
+
_write_json(HIDDEN_TESTS_DIR / "cross_sheet_lookup_02.json", {
|
| 525 |
+
"scenario_id": "cross_sheet_lookup_02",
|
| 526 |
+
"checks": [
|
| 527 |
+
{"sheet": "Payroll", "range": "B2:H26", "check": "no_blanks"},
|
| 528 |
+
],
|
| 529 |
+
"target_regions": [{"sheet": "Payroll", "range": "B2:H26"}],
|
| 530 |
+
})
|
| 531 |
+
|
| 532 |
+
|
| 533 |
+
# ══════════════════════════════════════════════════════════════════════
|
| 534 |
+
# Scenario 5: messy_table_extraction_01 — Vendor Invoice Processing
|
| 535 |
+
# ══════════════════════════════════════════════════════════════════════
|
| 536 |
+
|
| 537 |
+
def gen_messy_table_extraction_01():
|
| 538 |
+
wb = openpyxl.Workbook()
|
| 539 |
+
|
| 540 |
+
ws_raw = wb.active
|
| 541 |
+
ws_raw.title = "Raw_Invoices"
|
| 542 |
+
|
| 543 |
+
# Messy layout: header row at row 3 (rows 1-2 are title), blank rows between sections
|
| 544 |
+
ws_raw.cell(row=1, column=1, value="ACME Corp — Invoice Register 2024").font = Font(bold=True, size=14)
|
| 545 |
+
ws_raw.merge_cells("A1:F1")
|
| 546 |
+
ws_raw.cell(row=2, column=1, value="Exported from legacy system on 2024-06-15")
|
| 547 |
+
|
| 548 |
+
headers = ["Invoice #", "Date", "Vendor", "Amount", "Currency", "Status"]
|
| 549 |
+
for c, h in enumerate(headers, 1):
|
| 550 |
+
ws_raw.cell(row=3, column=c, value=h).font = _bold_font()
|
| 551 |
+
|
| 552 |
+
vendors = ["TechSupply Co.", "Office Depot", "CloudHost Inc.", "DataPipe LLC", "SecureNet Corp"]
|
| 553 |
+
statuses = ["Paid", "Pending", "Overdue", "Paid", "Pending"]
|
| 554 |
+
row = 4
|
| 555 |
+
invoice_count = 0
|
| 556 |
+
for section in range(4):
|
| 557 |
+
# Section header
|
| 558 |
+
ws_raw.cell(row=row, column=1, value=f"--- Q{section+1} 2024 ---").font = Font(italic=True)
|
| 559 |
+
row += 1
|
| 560 |
+
for i in range(random.randint(12, 18)):
|
| 561 |
+
inv_num = f"INV-2024-{invoice_count+1:04d}"
|
| 562 |
+
d = date(2024, section * 3 + 1, 1) + timedelta(days=random.randint(0, 85))
|
| 563 |
+
# Mix date formats deliberately
|
| 564 |
+
if i % 3 == 0:
|
| 565 |
+
date_str = d.strftime("%m/%d/%Y") # US format
|
| 566 |
+
elif i % 3 == 1:
|
| 567 |
+
date_str = d.strftime("%d-%m-%Y") # EU format
|
| 568 |
+
else:
|
| 569 |
+
date_str = d.isoformat() # ISO format
|
| 570 |
+
|
| 571 |
+
ws_raw.cell(row=row, column=1, value=inv_num)
|
| 572 |
+
ws_raw.cell(row=row, column=2, value=date_str)
|
| 573 |
+
ws_raw.cell(row=row, column=3, value=vendors[i % len(vendors)])
|
| 574 |
+
ws_raw.cell(row=row, column=4, value=round(random.uniform(500, 50000), 2))
|
| 575 |
+
ws_raw.cell(row=row, column=5, value="USD")
|
| 576 |
+
ws_raw.cell(row=row, column=6, value=statuses[i % len(statuses)])
|
| 577 |
+
row += 1
|
| 578 |
+
invoice_count += 1
|
| 579 |
+
# Blank separator
|
| 580 |
+
row += 1
|
| 581 |
+
|
| 582 |
+
# -- Processed sheet (target) --
|
| 583 |
+
ws_proc = wb.create_sheet("Processed")
|
| 584 |
+
proc_headers = ["Invoice #", "Date", "Vendor", "Amount", "Status"]
|
| 585 |
+
for c, h in enumerate(proc_headers, 1):
|
| 586 |
+
cell = ws_proc.cell(row=1, column=c, value=h)
|
| 587 |
+
cell.font = _header_font()
|
| 588 |
+
cell.fill = _header_fill()
|
| 589 |
+
|
| 590 |
+
# -- Vendor Lookup --
|
| 591 |
+
ws_vendor = wb.create_sheet("Vendor_Lookup")
|
| 592 |
+
ws_vendor.cell(row=1, column=1, value="Vendor Name").font = _bold_font()
|
| 593 |
+
ws_vendor.cell(row=1, column=2, value="Category").font = _bold_font()
|
| 594 |
+
ws_vendor.cell(row=1, column=3, value="Payment Terms").font = _bold_font()
|
| 595 |
+
vendor_cats = [
|
| 596 |
+
("TechSupply Co.", "Hardware", "Net 30"),
|
| 597 |
+
("Office Depot", "Supplies", "Net 15"),
|
| 598 |
+
("CloudHost Inc.", "Cloud", "Net 30"),
|
| 599 |
+
("DataPipe LLC", "Data", "Net 45"),
|
| 600 |
+
("SecureNet Corp", "Security", "Net 30"),
|
| 601 |
+
]
|
| 602 |
+
for i, (v, c, t) in enumerate(vendor_cats):
|
| 603 |
+
ws_vendor.cell(row=i + 2, column=1, value=v)
|
| 604 |
+
ws_vendor.cell(row=i + 2, column=2, value=c)
|
| 605 |
+
ws_vendor.cell(row=i + 2, column=3, value=t)
|
| 606 |
+
|
| 607 |
+
wb.save(TEMPLATES_DIR / "vendor_invoice_processing.xlsx")
|
| 608 |
+
|
| 609 |
+
_write_json(SCENARIOS_DIR / "messy_table_extraction_01.json", {
|
| 610 |
+
"id": "messy_table_extraction_01",
|
| 611 |
+
"description": "Extract and clean invoice data from a messy raw export with mixed date formats, section headers mixed in with data rows, and blank separator rows. All dates must be normalized to ISO format.",
|
| 612 |
+
"instructions": "The Raw_Invoices sheet has messy data exported from a legacy system: title rows at top, section header rows (like '--- Q1 2024 ---') mixed in with data, blank separator rows, and inconsistent date formats (MM/DD/YYYY, DD-MM-YYYY, and ISO). Extract all actual invoice rows into the Processed sheet with: (1) Invoice #, (2) Date in ISO format (YYYY-MM-DD), (3) Vendor, (4) Amount, (5) Status. Skip section headers and blank rows. Dates must all be converted to ISO format.",
|
| 613 |
+
"workbook": "vendor_invoice_processing.xlsx",
|
| 614 |
+
"max_steps": 60,
|
| 615 |
+
"category": "messy_table_extraction",
|
| 616 |
+
})
|
| 617 |
+
|
| 618 |
+
_write_json(HIDDEN_TESTS_DIR / "messy_table_extraction_01.json", {
|
| 619 |
+
"scenario_id": "messy_table_extraction_01",
|
| 620 |
+
"checks": [
|
| 621 |
+
{"sheet": "Processed", "check": "row_count_equals", "value": invoice_count},
|
| 622 |
+
{"sheet": "Processed", "column": "B", "check": "all_dates_iso_format"},
|
| 623 |
+
{"sheet": "Processed", "range": f"A2:E{invoice_count + 1}", "check": "no_blanks"},
|
| 624 |
+
],
|
| 625 |
+
"target_regions": [{"sheet": "Processed", "range": f"A2:E{invoice_count + 1}"}],
|
| 626 |
+
})
|
| 627 |
+
|
| 628 |
+
|
| 629 |
+
# ══════════════════════════════════════════════════════════════════════
|
| 630 |
+
# Scenario 6: schedule_grid_fill_01 — Employee Schedule Planning
|
| 631 |
+
# ══════════════════════════════════════════════════════════════════════
|
| 632 |
+
|
| 633 |
+
def gen_schedule_grid_fill_01():
|
| 634 |
+
wb = openpyxl.Workbook()
|
| 635 |
+
|
| 636 |
+
employees = NAMES[:12]
|
| 637 |
+
days = ["Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday", "Sunday"]
|
| 638 |
+
|
| 639 |
+
# -- Constraints sheet --
|
| 640 |
+
ws_con = wb.active
|
| 641 |
+
ws_con.title = "Constraints"
|
| 642 |
+
ws_con.cell(row=1, column=1, value="Scheduling Constraints").font = Font(bold=True, size=12)
|
| 643 |
+
constraints = [
|
| 644 |
+
"No employee works more than 5 days per week.",
|
| 645 |
+
"Night shift (N) must not be followed by Morning shift (M) the next day.",
|
| 646 |
+
"At least 2 employees must be on Morning shift every day.",
|
| 647 |
+
"At least 1 employee must be on Night shift every day.",
|
| 648 |
+
"Each employee must work at least 3 days per week.",
|
| 649 |
+
"Saturday and Sunday must have at least 3 employees on Afternoon shift.",
|
| 650 |
+
]
|
| 651 |
+
for i, c in enumerate(constraints):
|
| 652 |
+
ws_con.cell(row=i + 3, column=1, value=c)
|
| 653 |
+
|
| 654 |
+
# -- Availability sheet (exceptions) --
|
| 655 |
+
ws_avail = wb.create_sheet("Availability")
|
| 656 |
+
ws_avail.cell(row=1, column=1, value="Employee").font = _bold_font()
|
| 657 |
+
ws_avail.cell(row=1, column=2, value="Unavailable Day").font = _bold_font()
|
| 658 |
+
ws_avail.cell(row=1, column=3, value="Reason").font = _bold_font()
|
| 659 |
+
exceptions = [
|
| 660 |
+
(employees[0], "Monday", "PTO"),
|
| 661 |
+
(employees[0], "Tuesday", "PTO"),
|
| 662 |
+
(employees[3], "Saturday", "Personal"),
|
| 663 |
+
(employees[5], "Sunday", "Religious"),
|
| 664 |
+
(employees[7], "Friday", "Medical"),
|
| 665 |
+
(employees[9], "Wednesday", "Training"),
|
| 666 |
+
(employees[11], "Thursday", "Court duty"),
|
| 667 |
+
]
|
| 668 |
+
for i, (emp, day, reason) in enumerate(exceptions):
|
| 669 |
+
ws_avail.cell(row=i + 2, column=1, value=emp)
|
| 670 |
+
ws_avail.cell(row=i + 2, column=2, value=day)
|
| 671 |
+
ws_avail.cell(row=i + 2, column=3, value=reason)
|
| 672 |
+
|
| 673 |
+
# -- Output sheet (empty grid) --
|
| 674 |
+
ws_out = wb.create_sheet("Output")
|
| 675 |
+
ws_out.cell(row=1, column=1, value="Employee").font = _bold_font()
|
| 676 |
+
for j, day in enumerate(days):
|
| 677 |
+
ws_out.cell(row=1, column=j + 2, value=day).font = _bold_font()
|
| 678 |
+
for i, emp in enumerate(employees):
|
| 679 |
+
ws_out.cell(row=i + 2, column=1, value=emp)
|
| 680 |
+
for j in range(len(days)):
|
| 681 |
+
ws_out.cell(row=i + 2, column=j + 2).fill = _yellow_fill()
|
| 682 |
+
|
| 683 |
+
# -- Reference (shift codes) --
|
| 684 |
+
ws_ref = wb.create_sheet("Shift_Codes")
|
| 685 |
+
ws_ref.cell(row=1, column=1, value="Code").font = _bold_font()
|
| 686 |
+
ws_ref.cell(row=1, column=2, value="Shift").font = _bold_font()
|
| 687 |
+
ws_ref.cell(row=1, column=3, value="Hours").font = _bold_font()
|
| 688 |
+
codes = [("M", "Morning", "6:00-14:00"), ("A", "Afternoon", "14:00-22:00"), ("N", "Night", "22:00-6:00"), ("X", "Off", "N/A")]
|
| 689 |
+
for i, (code, shift, hours) in enumerate(codes):
|
| 690 |
+
ws_ref.cell(row=i + 2, column=1, value=code)
|
| 691 |
+
ws_ref.cell(row=i + 2, column=2, value=shift)
|
| 692 |
+
ws_ref.cell(row=i + 2, column=3, value=hours)
|
| 693 |
+
|
| 694 |
+
wb.save(TEMPLATES_DIR / "employee_schedule_grid.xlsx")
|
| 695 |
+
|
| 696 |
+
_write_json(SCENARIOS_DIR / "schedule_grid_fill_01.json", {
|
| 697 |
+
"id": "schedule_grid_fill_01",
|
| 698 |
+
"description": "Fill an employee schedule grid for 12 employees across 7 days, respecting prose constraints on max days, shift transitions, minimum coverage, and availability exceptions.",
|
| 699 |
+
"instructions": "Fill the Output sheet with shift codes (M=Morning, A=Afternoon, N=Night, X=Off) for each employee and day. You must satisfy ALL constraints from the Constraints sheet and respect the availability exceptions from the Availability sheet. Unavailable employees must have X for that day. Check the Shift_Codes sheet for valid codes.",
|
| 700 |
+
"workbook": "employee_schedule_grid.xlsx",
|
| 701 |
+
"max_steps": 70,
|
| 702 |
+
"category": "schedule_grid_fill",
|
| 703 |
+
})
|
| 704 |
+
|
| 705 |
+
_write_json(HIDDEN_TESTS_DIR / "schedule_grid_fill_01.json", {
|
| 706 |
+
"scenario_id": "schedule_grid_fill_01",
|
| 707 |
+
"checks": [
|
| 708 |
+
{"sheet": "Output", "range": "B2:H13", "check": "no_blanks"},
|
| 709 |
+
{"sheet": "Output", "check": "constraint_satisfaction", "constraints_sheet": "Constraints"},
|
| 710 |
+
],
|
| 711 |
+
"target_regions": [{"sheet": "Output", "range": "B2:H13"}],
|
| 712 |
+
})
|
| 713 |
+
|
| 714 |
+
|
| 715 |
+
# ══════════════════════════════════════════════════════════════════════
|
| 716 |
+
# Scenario 7: ledger_reconciliation_01 — Bank Statement Reconciliation
|
| 717 |
+
# ══════════════════════════════════════════════════════════════════════
|
| 718 |
+
|
| 719 |
+
def gen_ledger_reconciliation_01():
|
| 720 |
+
wb = openpyxl.Workbook()
|
| 721 |
+
|
| 722 |
+
# -- Bank Statement --
|
| 723 |
+
ws_bank = wb.active
|
| 724 |
+
ws_bank.title = "Bank_Statement"
|
| 725 |
+
for c, h in enumerate(["Date", "Description", "Reference", "Amount", "Balance"], 1):
|
| 726 |
+
ws_bank.cell(row=1, column=c, value=h).font = _bold_font()
|
| 727 |
+
|
| 728 |
+
bank_txns = []
|
| 729 |
+
balance = 50000
|
| 730 |
+
for i in range(60):
|
| 731 |
+
d = date(2024, 1, 1) + timedelta(days=random.randint(0, 180))
|
| 732 |
+
desc = random.choice(["Wire Transfer", "ACH Payment", "Check #" + str(random.randint(1000, 9999)),
|
| 733 |
+
"Deposit", "Service Fee", "Interest Credit"])
|
| 734 |
+
ref = f"BNK-{random.randint(10000, 99999)}"
|
| 735 |
+
amt = round(random.uniform(-5000, 10000), 2)
|
| 736 |
+
balance += amt
|
| 737 |
+
bank_txns.append((d, desc, ref, amt, round(balance, 2)))
|
| 738 |
+
|
| 739 |
+
bank_txns.sort(key=lambda x: x[0])
|
| 740 |
+
for i, (d, desc, ref, amt, bal) in enumerate(bank_txns):
|
| 741 |
+
r = i + 2
|
| 742 |
+
ws_bank.cell(row=r, column=1, value=d)
|
| 743 |
+
ws_bank.cell(row=r, column=2, value=desc)
|
| 744 |
+
ws_bank.cell(row=r, column=3, value=ref)
|
| 745 |
+
ws_bank.cell(row=r, column=4, value=amt)
|
| 746 |
+
ws_bank.cell(row=r, column=5, value=bal)
|
| 747 |
+
|
| 748 |
+
# -- Internal Ledger (slightly different — some missing, some extra, some amount mismatches) --
|
| 749 |
+
ws_ledger = wb.active if wb.active.title == "Bank_Statement" else wb.create_sheet("Internal_Ledger")
|
| 750 |
+
ws_ledger = wb.create_sheet("Internal_Ledger")
|
| 751 |
+
for c, h in enumerate(["Date", "Description", "GL Code", "Amount", "Reconciled"], 1):
|
| 752 |
+
ws_ledger.cell(row=1, column=c, value=h).font = _bold_font()
|
| 753 |
+
|
| 754 |
+
ledger_row = 2
|
| 755 |
+
matched = 0
|
| 756 |
+
unmatched_bank = []
|
| 757 |
+
for i, (d, desc, ref, amt, bal) in enumerate(bank_txns):
|
| 758 |
+
if random.random() < 0.1:
|
| 759 |
+
unmatched_bank.append(i)
|
| 760 |
+
continue
|
| 761 |
+
gl = f"GL-{random.randint(4000, 4999)}"
|
| 762 |
+
ledger_amt = amt
|
| 763 |
+
if random.random() < 0.08:
|
| 764 |
+
ledger_amt = round(amt + random.uniform(-50, 50), 2)
|
| 765 |
+
ws_ledger.cell(row=ledger_row, column=1, value=d)
|
| 766 |
+
ws_ledger.cell(row=ledger_row, column=2, value=desc)
|
| 767 |
+
ws_ledger.cell(row=ledger_row, column=3, value=gl)
|
| 768 |
+
ws_ledger.cell(row=ledger_row, column=4, value=ledger_amt)
|
| 769 |
+
ws_ledger.cell(row=ledger_row, column=5, value="No")
|
| 770 |
+
ledger_row += 1
|
| 771 |
+
matched += 1
|
| 772 |
+
|
| 773 |
+
# Add some extra ledger entries not in bank
|
| 774 |
+
for i in range(5):
|
| 775 |
+
d = date(2024, 1, 1) + timedelta(days=random.randint(0, 180))
|
| 776 |
+
ws_ledger.cell(row=ledger_row, column=1, value=d)
|
| 777 |
+
ws_ledger.cell(row=ledger_row, column=2, value=f"Manual Adjustment {i+1}")
|
| 778 |
+
ws_ledger.cell(row=ledger_row, column=3, value=f"GL-{random.randint(5000, 5999)}")
|
| 779 |
+
ws_ledger.cell(row=ledger_row, column=4, value=round(random.uniform(-2000, 2000), 2))
|
| 780 |
+
ws_ledger.cell(row=ledger_row, column=5, value="No")
|
| 781 |
+
ledger_row += 1
|
| 782 |
+
|
| 783 |
+
# -- Reconciled sheet (target) --
|
| 784 |
+
ws_recon = wb.create_sheet("Reconciled")
|
| 785 |
+
for c, h in enumerate(["Date", "Description", "Bank Amount", "Ledger Amount", "Difference", "Status"], 1):
|
| 786 |
+
cell = ws_recon.cell(row=1, column=c, value=h)
|
| 787 |
+
cell.font = _header_font()
|
| 788 |
+
cell.fill = _header_fill()
|
| 789 |
+
|
| 790 |
+
wb.save(TEMPLATES_DIR / "bank_reconciliation.xlsx")
|
| 791 |
+
|
| 792 |
+
_write_json(SCENARIOS_DIR / "ledger_reconciliation_01.json", {
|
| 793 |
+
"id": "ledger_reconciliation_01",
|
| 794 |
+
"description": "Reconcile a bank statement against an internal ledger. Find mismatches, missing entries, and amount discrepancies. Fill the Reconciled sheet.",
|
| 795 |
+
"instructions": "Compare Bank_Statement and Internal_Ledger to produce a reconciliation report in the Reconciled sheet. For each transaction: match by date and description. Record the Bank Amount, Ledger Amount, Difference (Bank - Ledger), and Status (Matched/Mismatch/Bank Only/Ledger Only). Include ALL transactions from both sources. Sort by date.",
|
| 796 |
+
"workbook": "bank_reconciliation.xlsx",
|
| 797 |
+
"max_steps": 60,
|
| 798 |
+
"category": "ledger_reconciliation",
|
| 799 |
+
})
|
| 800 |
+
|
| 801 |
+
total_entries = len(bank_txns) + 5
|
| 802 |
+
_write_json(HIDDEN_TESTS_DIR / "ledger_reconciliation_01.json", {
|
| 803 |
+
"scenario_id": "ledger_reconciliation_01",
|
| 804 |
+
"checks": [
|
| 805 |
+
{"sheet": "Reconciled", "range": f"A2:F{total_entries + 1}", "check": "no_blanks"},
|
| 806 |
+
{"sheet": "Reconciled", "column": "A", "check": "all_dates_iso_format"},
|
| 807 |
+
],
|
| 808 |
+
"target_regions": [{"sheet": "Reconciled", "range": f"A2:F{total_entries + 1}"}],
|
| 809 |
+
})
|
| 810 |
+
|
| 811 |
+
|
| 812 |
+
# ══════════════════════════════════════════════════════════════════════
|
| 813 |
+
# Scenario 8: ledger_reconciliation_02 — Multi-Currency Reconciliation
|
| 814 |
+
# ══════════════════════════════════════════════════════════════════════
|
| 815 |
+
|
| 816 |
+
def gen_ledger_reconciliation_02():
|
| 817 |
+
wb = openpyxl.Workbook()
|
| 818 |
+
|
| 819 |
+
# -- Transactions USD --
|
| 820 |
+
ws_usd = wb.active
|
| 821 |
+
ws_usd.title = "Transactions_USD"
|
| 822 |
+
for c, h in enumerate(["Date", "Description", "Amount USD"], 1):
|
| 823 |
+
ws_usd.cell(row=1, column=c, value=h).font = _bold_font()
|
| 824 |
+
usd_total = 0
|
| 825 |
+
for i in range(30):
|
| 826 |
+
r = i + 2
|
| 827 |
+
d = date(2024, 1, 1) + timedelta(days=random.randint(0, 180))
|
| 828 |
+
amt = round(random.uniform(100, 15000), 2)
|
| 829 |
+
usd_total += amt
|
| 830 |
+
ws_usd.cell(row=r, column=1, value=d.strftime("%m/%d/%Y"))
|
| 831 |
+
ws_usd.cell(row=r, column=2, value=f"USD Transaction {i+1}")
|
| 832 |
+
ws_usd.cell(row=r, column=3, value=amt)
|
| 833 |
+
|
| 834 |
+
# -- Transactions EUR --
|
| 835 |
+
ws_eur = wb.create_sheet("Transactions_EUR")
|
| 836 |
+
for c, h in enumerate(["Date", "Description", "Amount EUR"], 1):
|
| 837 |
+
ws_eur.cell(row=1, column=c, value=h).font = _bold_font()
|
| 838 |
+
eur_amounts = []
|
| 839 |
+
for i in range(20):
|
| 840 |
+
r = i + 2
|
| 841 |
+
d = date(2024, 1, 1) + timedelta(days=random.randint(0, 180))
|
| 842 |
+
amt = round(random.uniform(100, 12000), 2)
|
| 843 |
+
eur_amounts.append(amt)
|
| 844 |
+
ws_eur.cell(row=r, column=1, value=d.strftime("%d-%m-%Y"))
|
| 845 |
+
ws_eur.cell(row=r, column=2, value=f"EUR Transaction {i+1}")
|
| 846 |
+
ws_eur.cell(row=r, column=3, value=amt)
|
| 847 |
+
|
| 848 |
+
# -- Exchange Rates --
|
| 849 |
+
ws_fx = wb.create_sheet("Exchange_Rates")
|
| 850 |
+
ws_fx.cell(row=1, column=1, value="Month").font = _bold_font()
|
| 851 |
+
ws_fx.cell(row=1, column=2, value="EUR_to_USD").font = _bold_font()
|
| 852 |
+
months = ["January", "February", "March", "April", "May", "June"]
|
| 853 |
+
rates = [1.08, 1.09, 1.07, 1.10, 1.08, 1.11]
|
| 854 |
+
for i, (m, rate) in enumerate(zip(months, rates)):
|
| 855 |
+
ws_fx.cell(row=i + 2, column=1, value=m)
|
| 856 |
+
ws_fx.cell(row=i + 2, column=2, value=rate)
|
| 857 |
+
|
| 858 |
+
# -- Summary (target) --
|
| 859 |
+
ws_sum = wb.create_sheet("Summary")
|
| 860 |
+
ws_sum.cell(row=1, column=1, value="Multi-Currency Reconciliation Summary").font = Font(bold=True, size=14)
|
| 861 |
+
|
| 862 |
+
ws_sum.cell(row=3, column=1, value="Category").font = _bold_font()
|
| 863 |
+
ws_sum.cell(row=3, column=2, value="Amount (USD)").font = _bold_font()
|
| 864 |
+
|
| 865 |
+
ws_sum.cell(row=4, column=1, value="Total USD Transactions")
|
| 866 |
+
ws_sum.cell(row=4, column=2).fill = _yellow_fill()
|
| 867 |
+
|
| 868 |
+
ws_sum.cell(row=5, column=1, value="Total EUR Transactions (converted to USD)")
|
| 869 |
+
ws_sum.cell(row=5, column=2).fill = _yellow_fill()
|
| 870 |
+
|
| 871 |
+
ws_sum.cell(row=6, column=1, value="Grand Total (USD)")
|
| 872 |
+
ws_sum.cell(row=6, column=2).fill = _yellow_fill()
|
| 873 |
+
|
| 874 |
+
ws_sum.cell(row=8, column=1, value="EUR Transaction Count")
|
| 875 |
+
ws_sum.cell(row=8, column=2).fill = _yellow_fill()
|
| 876 |
+
|
| 877 |
+
ws_sum.cell(row=9, column=1, value="USD Transaction Count")
|
| 878 |
+
ws_sum.cell(row=9, column=2).fill = _yellow_fill()
|
| 879 |
+
|
| 880 |
+
ws_sum.cell(row=10, column=1, value="Total Transaction Count")
|
| 881 |
+
ws_sum.cell(row=10, column=2).fill = _yellow_fill()
|
| 882 |
+
|
| 883 |
+
wb.save(TEMPLATES_DIR / "multi_currency_reconciliation.xlsx")
|
| 884 |
+
|
| 885 |
+
avg_rate = sum(rates) / len(rates)
|
| 886 |
+
eur_in_usd = round(sum(eur_amounts) * avg_rate, 2)
|
| 887 |
+
|
| 888 |
+
_write_json(SCENARIOS_DIR / "ledger_reconciliation_02.json", {
|
| 889 |
+
"id": "ledger_reconciliation_02",
|
| 890 |
+
"description": "Reconcile USD and EUR transaction sheets into a unified summary. EUR dates are in DD-MM-YYYY format, USD dates in MM/DD/YYYY. Convert EUR to USD using the Exchange_Rates sheet.",
|
| 891 |
+
"instructions": "The workbook has USD and EUR transaction sheets with different date formats. Convert all EUR transactions to USD using the monthly exchange rate from the Exchange_Rates sheet (match each transaction's month to the correct rate). Fill the Summary sheet with: Total USD Transactions, Total EUR Transactions converted to USD, Grand Total, and transaction counts. Dates in the transaction sheets use different formats — be careful when determining which month each EUR transaction falls in.",
|
| 892 |
+
"workbook": "multi_currency_reconciliation.xlsx",
|
| 893 |
+
"max_steps": 55,
|
| 894 |
+
"category": "ledger_reconciliation",
|
| 895 |
+
})
|
| 896 |
+
|
| 897 |
+
_write_json(HIDDEN_TESTS_DIR / "ledger_reconciliation_02.json", {
|
| 898 |
+
"scenario_id": "ledger_reconciliation_02",
|
| 899 |
+
"checks": [
|
| 900 |
+
{"sheet": "Summary", "cell": "B4", "expected_value_range": [usd_total * 0.99, usd_total * 1.01]},
|
| 901 |
+
{"sheet": "Summary", "cell": "B9", "expected_value_range": [30, 30]},
|
| 902 |
+
{"sheet": "Summary", "cell": "B8", "expected_value_range": [20, 20]},
|
| 903 |
+
{"sheet": "Summary", "cell": "B10", "expected_value_range": [50, 50]},
|
| 904 |
+
{"sheet": "Summary", "range": "B4:B10", "check": "no_blanks"},
|
| 905 |
+
],
|
| 906 |
+
"target_regions": [{"sheet": "Summary", "range": "B4:B10"}],
|
| 907 |
+
})
|
| 908 |
+
|
| 909 |
+
|
| 910 |
+
# ══════════════════════════════════════════════════════════════════════
|
| 911 |
+
# Scenario 9: range_transformation_01 — Data Pivot and Reshape
|
| 912 |
+
# ══════════════════════════════════════════════════════════════════════
|
| 913 |
+
|
| 914 |
+
def gen_range_transformation_01():
|
| 915 |
+
wb = openpyxl.Workbook()
|
| 916 |
+
|
| 917 |
+
# -- Raw Data (long format) --
|
| 918 |
+
ws_raw = wb.active
|
| 919 |
+
ws_raw.title = "Raw_Data"
|
| 920 |
+
for c, h in enumerate(["Employee", "Month", "Metric", "Value"], 1):
|
| 921 |
+
ws_raw.cell(row=1, column=c, value=h).font = _bold_font()
|
| 922 |
+
|
| 923 |
+
emps = NAMES[:8]
|
| 924 |
+
months_list = ["Jan", "Feb", "Mar", "Apr", "May", "Jun"]
|
| 925 |
+
metrics = ["Sales", "Returns", "Net Revenue"]
|
| 926 |
+
|
| 927 |
+
row = 2
|
| 928 |
+
data_map = {}
|
| 929 |
+
for emp in emps:
|
| 930 |
+
for month in months_list:
|
| 931 |
+
sales = random.randint(10000, 80000)
|
| 932 |
+
returns = random.randint(500, sales // 5)
|
| 933 |
+
net = sales - returns
|
| 934 |
+
for metric, val in [("Sales", sales), ("Returns", returns), ("Net Revenue", net)]:
|
| 935 |
+
ws_raw.cell(row=row, column=1, value=emp)
|
| 936 |
+
ws_raw.cell(row=row, column=2, value=month)
|
| 937 |
+
ws_raw.cell(row=row, column=3, value=metric)
|
| 938 |
+
ws_raw.cell(row=row, column=4, value=val)
|
| 939 |
+
data_map[(emp, month, metric)] = val
|
| 940 |
+
row += 1
|
| 941 |
+
|
| 942 |
+
# -- Instructions --
|
| 943 |
+
ws_instr = wb.create_sheet("Instructions")
|
| 944 |
+
ws_instr.cell(row=1, column=1, value="Data Transformation Task").font = Font(bold=True, size=12)
|
| 945 |
+
ws_instr.cell(row=3, column=1, value="Pivot the Raw_Data into the Pivot_Output sheet.")
|
| 946 |
+
ws_instr.cell(row=4, column=1, value="Layout: Rows = Employees, Column groups = Months")
|
| 947 |
+
ws_instr.cell(row=5, column=1, value="Under each month, show three sub-columns: Sales, Returns, Net Revenue")
|
| 948 |
+
ws_instr.cell(row=6, column=1, value="Add a final column 'Total Net Revenue' summing Net Revenue across all months")
|
| 949 |
+
ws_instr.cell(row=7, column=1, value="Sort employees alphabetically.")
|
| 950 |
+
|
| 951 |
+
# -- Pivot Output (target, headers pre-filled) --
|
| 952 |
+
ws_pivot = wb.create_sheet("Pivot_Output")
|
| 953 |
+
ws_pivot.cell(row=1, column=1, value="Employee").font = _bold_font()
|
| 954 |
+
col = 2
|
| 955 |
+
for month in months_list:
|
| 956 |
+
ws_pivot.cell(row=1, column=col, value=f"{month} Sales").font = _bold_font()
|
| 957 |
+
ws_pivot.cell(row=1, column=col + 1, value=f"{month} Returns").font = _bold_font()
|
| 958 |
+
ws_pivot.cell(row=1, column=col + 2, value=f"{month} Net Revenue").font = _bold_font()
|
| 959 |
+
col += 3
|
| 960 |
+
ws_pivot.cell(row=1, column=col, value="Total Net Revenue").font = _bold_font()
|
| 961 |
+
|
| 962 |
+
for i, emp in enumerate(sorted(emps)):
|
| 963 |
+
ws_pivot.cell(row=i + 2, column=1, value=emp)
|
| 964 |
+
for c in range(2, col + 1):
|
| 965 |
+
ws_pivot.cell(row=i + 2, column=c).fill = _yellow_fill()
|
| 966 |
+
|
| 967 |
+
wb.save(TEMPLATES_DIR / "data_pivot_reshape.xlsx")
|
| 968 |
+
|
| 969 |
+
# Calculate expected total net revenues for checks
|
| 970 |
+
sorted_emps = sorted(emps)
|
| 971 |
+
checks = []
|
| 972 |
+
for i, emp in enumerate(sorted_emps):
|
| 973 |
+
total_net = sum(data_map.get((emp, m, "Net Revenue"), 0) for m in months_list)
|
| 974 |
+
checks.append({
|
| 975 |
+
"sheet": "Pivot_Output",
|
| 976 |
+
"cell": f"{get_column_letter(col)}{i+2}",
|
| 977 |
+
"expected_value_range": [total_net * 0.99, total_net * 1.01],
|
| 978 |
+
})
|
| 979 |
+
checks.append({"sheet": "Pivot_Output", "range": f"B2:{get_column_letter(col)}{len(sorted_emps)+1}", "check": "no_blanks"})
|
| 980 |
+
checks.append({"sheet": "Pivot_Output", "check": "row_count_equals", "value": len(sorted_emps)})
|
| 981 |
+
|
| 982 |
+
_write_json(SCENARIOS_DIR / "range_transformation_01.json", {
|
| 983 |
+
"id": "range_transformation_01",
|
| 984 |
+
"description": "Pivot long-format employee metrics data into a wide-format table. Each employee gets one row with Sales, Returns, and Net Revenue for each of 6 months, plus a total.",
|
| 985 |
+
"instructions": "The Raw_Data sheet has employee performance metrics in long format (one row per employee-month-metric combination). Pivot this into the Pivot_Output sheet: one row per employee (sorted alphabetically), with columns for each month's Sales, Returns, and Net Revenue. Add a Total Net Revenue column at the end. The headers are pre-filled; fill the data cells.",
|
| 986 |
+
"workbook": "data_pivot_reshape.xlsx",
|
| 987 |
+
"max_steps": 60,
|
| 988 |
+
"category": "range_transformation",
|
| 989 |
+
})
|
| 990 |
+
|
| 991 |
+
_write_json(HIDDEN_TESTS_DIR / "range_transformation_01.json", {
|
| 992 |
+
"scenario_id": "range_transformation_01",
|
| 993 |
+
"checks": checks,
|
| 994 |
+
"target_regions": [{"sheet": "Pivot_Output", "range": f"B2:{get_column_letter(col)}{len(sorted_emps)+1}"}],
|
| 995 |
+
})
|
| 996 |
+
|
| 997 |
+
|
| 998 |
+
# ══════════════════════════════════════════════════════════════════════
|
| 999 |
+
# Scenario 10: conditional_aggregation_01 — Sales Commission
|
| 1000 |
+
# ══════════════════════════════════════════════════════════════════════
|
| 1001 |
+
|
| 1002 |
+
def gen_conditional_aggregation_01():
|
| 1003 |
+
wb = openpyxl.Workbook()
|
| 1004 |
+
|
| 1005 |
+
# -- Sales data --
|
| 1006 |
+
ws_sales = wb.active
|
| 1007 |
+
ws_sales.title = "Sales"
|
| 1008 |
+
for c, h in enumerate(["Salesperson", "Region", "Q1", "Q2", "Q3", "Q4", "Annual Total"], 1):
|
| 1009 |
+
ws_sales.cell(row=1, column=c, value=h).font = _bold_font()
|
| 1010 |
+
|
| 1011 |
+
salespeople = NAMES[:15]
|
| 1012 |
+
sales_data = {}
|
| 1013 |
+
for i, sp in enumerate(salespeople):
|
| 1014 |
+
r = i + 2
|
| 1015 |
+
ws_sales.cell(row=r, column=1, value=sp)
|
| 1016 |
+
ws_sales.cell(row=r, column=2, value=REGIONS[i % len(REGIONS)])
|
| 1017 |
+
quarterly = [random.randint(20000, 150000) for _ in range(4)]
|
| 1018 |
+
for q in range(4):
|
| 1019 |
+
ws_sales.cell(row=r, column=q + 3, value=quarterly[q])
|
| 1020 |
+
ws_sales.cell(row=r, column=7, value=f"=SUM(C{r}:F{r})")
|
| 1021 |
+
sales_data[sp] = sum(quarterly)
|
| 1022 |
+
|
| 1023 |
+
# -- Commission Rules (tiered, complex) --
|
| 1024 |
+
ws_rules = wb.create_sheet("Commission_Rules")
|
| 1025 |
+
ws_rules.cell(row=1, column=1, value="Commission Tier Structure").font = Font(bold=True, size=12)
|
| 1026 |
+
|
| 1027 |
+
ws_rules.cell(row=3, column=1, value="Tier").font = _bold_font()
|
| 1028 |
+
ws_rules.cell(row=3, column=2, value="Min Annual Sales").font = _bold_font()
|
| 1029 |
+
ws_rules.cell(row=3, column=3, value="Max Annual Sales").font = _bold_font()
|
| 1030 |
+
ws_rules.cell(row=3, column=4, value="Commission Rate").font = _bold_font()
|
| 1031 |
+
tiers = [
|
| 1032 |
+
("Bronze", 0, 200000, 0.03),
|
| 1033 |
+
("Silver", 200001, 400000, 0.05),
|
| 1034 |
+
("Gold", 400001, 600000, 0.08),
|
| 1035 |
+
("Platinum", 600001, 99999999, 0.12),
|
| 1036 |
+
]
|
| 1037 |
+
for i, (tier, mn, mx, rate) in enumerate(tiers):
|
| 1038 |
+
r = i + 4
|
| 1039 |
+
ws_rules.cell(row=r, column=1, value=tier)
|
| 1040 |
+
ws_rules.cell(row=r, column=2, value=mn)
|
| 1041 |
+
ws_rules.cell(row=r, column=3, value=mx)
|
| 1042 |
+
ws_rules.cell(row=r, column=4, value=rate)
|
| 1043 |
+
|
| 1044 |
+
ws_rules.cell(row=9, column=1, value="IMPORTANT: Commission is calculated on the FULL annual total,")
|
| 1045 |
+
ws_rules.cell(row=10, column=1, value="not just the amount within each tier. Apply the single rate for the tier.")
|
| 1046 |
+
ws_rules.cell(row=11, column=1, value="Regional bonus: West region gets +2% on top of tier rate.")
|
| 1047 |
+
|
| 1048 |
+
# -- Commissions (target) --
|
| 1049 |
+
ws_comm = wb.create_sheet("Commissions")
|
| 1050 |
+
for c, h in enumerate(["Salesperson", "Region", "Annual Sales", "Tier", "Base Rate", "Regional Bonus", "Total Rate", "Commission Amount"], 1):
|
| 1051 |
+
cell = ws_comm.cell(row=1, column=c, value=h)
|
| 1052 |
+
cell.font = _bold_font()
|
| 1053 |
+
for i, sp in enumerate(salespeople):
|
| 1054 |
+
r = i + 2
|
| 1055 |
+
ws_comm.cell(row=r, column=1, value=sp)
|
| 1056 |
+
for c in range(2, 9):
|
| 1057 |
+
ws_comm.cell(row=r, column=c).fill = _yellow_fill()
|
| 1058 |
+
|
| 1059 |
+
wb.save(TEMPLATES_DIR / "sales_commission.xlsx")
|
| 1060 |
+
|
| 1061 |
+
# Calculate expected values
|
| 1062 |
+
checks = []
|
| 1063 |
+
for i, sp in enumerate(salespeople):
|
| 1064 |
+
total = sales_data[sp]
|
| 1065 |
+
for tier_name, mn, mx, rate in tiers:
|
| 1066 |
+
if mn <= total <= mx:
|
| 1067 |
+
base_rate = rate
|
| 1068 |
+
break
|
| 1069 |
+
region = REGIONS[i % len(REGIONS)]
|
| 1070 |
+
regional_bonus = 0.02 if region == "West" else 0
|
| 1071 |
+
total_rate = base_rate + regional_bonus
|
| 1072 |
+
commission = round(total * total_rate, 2)
|
| 1073 |
+
r = i + 2
|
| 1074 |
+
checks.append({
|
| 1075 |
+
"sheet": "Commissions", "cell": f"H{r}",
|
| 1076 |
+
"expected_value_range": [commission * 0.99, commission * 1.01],
|
| 1077 |
+
})
|
| 1078 |
+
|
| 1079 |
+
checks.append({"sheet": "Commissions", "range": "B2:H16", "check": "no_blanks"})
|
| 1080 |
+
|
| 1081 |
+
_write_json(SCENARIOS_DIR / "conditional_aggregation_01.json", {
|
| 1082 |
+
"id": "conditional_aggregation_01",
|
| 1083 |
+
"description": "Calculate tiered sales commissions for 15 salespeople. Commission rates depend on annual total tier. West region gets a +2% bonus on top of the base tier rate.",
|
| 1084 |
+
"instructions": "Fill the Commissions sheet for each salesperson. Look up their Annual Sales from the Sales sheet, determine their tier from Commission_Rules, and calculate the commission. IMPORTANT: Read the Commission_Rules sheet carefully — the commission rate is applied to the FULL annual total (not marginal). The West region gets an additional +2% regional bonus. Fill all columns: Region, Annual Sales, Tier, Base Rate, Regional Bonus, Total Rate, Commission Amount.",
|
| 1085 |
+
"workbook": "sales_commission.xlsx",
|
| 1086 |
+
"max_steps": 55,
|
| 1087 |
+
"category": "conditional_aggregation",
|
| 1088 |
+
})
|
| 1089 |
+
|
| 1090 |
+
_write_json(HIDDEN_TESTS_DIR / "conditional_aggregation_01.json", {
|
| 1091 |
+
"scenario_id": "conditional_aggregation_01",
|
| 1092 |
+
"checks": checks,
|
| 1093 |
+
"target_regions": [{"sheet": "Commissions", "range": "B2:H16"}],
|
| 1094 |
+
})
|
| 1095 |
+
|
| 1096 |
+
|
| 1097 |
+
# ══════════════════════════════════════════════════════════════════════
|
| 1098 |
+
# Scenario 11: conditional_aggregation_02 — Budget Allocation
|
| 1099 |
+
# ══════════════════════════════════════════════════════════════════════
|
| 1100 |
+
|
| 1101 |
+
def gen_conditional_aggregation_02():
|
| 1102 |
+
wb = openpyxl.Workbook()
|
| 1103 |
+
|
| 1104 |
+
# -- Requests --
|
| 1105 |
+
ws_req = wb.active
|
| 1106 |
+
ws_req.title = "Requests"
|
| 1107 |
+
for c, h in enumerate(["Request ID", "Department", "Priority", "Requested Amount", "Justification"], 1):
|
| 1108 |
+
ws_req.cell(row=1, column=c, value=h).font = _bold_font()
|
| 1109 |
+
|
| 1110 |
+
priorities = ["Critical", "High", "Medium", "Low"]
|
| 1111 |
+
departments = ["Engineering", "Marketing", "Sales", "Operations", "HR"]
|
| 1112 |
+
requests_data = []
|
| 1113 |
+
for i in range(20):
|
| 1114 |
+
r = i + 2
|
| 1115 |
+
req_id = f"REQ-{i+1:03d}"
|
| 1116 |
+
dept = departments[i % len(departments)]
|
| 1117 |
+
pri = priorities[i % len(priorities)]
|
| 1118 |
+
amt = random.randint(5, 100) * 1000
|
| 1119 |
+
ws_req.cell(row=r, column=1, value=req_id)
|
| 1120 |
+
ws_req.cell(row=r, column=2, value=dept)
|
| 1121 |
+
ws_req.cell(row=r, column=3, value=pri)
|
| 1122 |
+
ws_req.cell(row=r, column=4, value=amt)
|
| 1123 |
+
ws_req.cell(row=r, column=5, value=f"Budget for {dept.lower()} initiative {i+1}")
|
| 1124 |
+
requests_data.append((req_id, dept, pri, amt))
|
| 1125 |
+
|
| 1126 |
+
# -- Budget Pool --
|
| 1127 |
+
ws_pool = wb.create_sheet("Budget_Pool")
|
| 1128 |
+
ws_pool.cell(row=1, column=1, value="Total Available Budget").font = _bold_font()
|
| 1129 |
+
total_budget = 800000
|
| 1130 |
+
ws_pool.cell(row=1, column=2, value=total_budget)
|
| 1131 |
+
|
| 1132 |
+
ws_pool.cell(row=3, column=1, value="Allocation Rules:").font = _bold_font()
|
| 1133 |
+
ws_pool.cell(row=4, column=1, value="1. Critical requests get 100% of requested amount (if budget allows).")
|
| 1134 |
+
ws_pool.cell(row=5, column=1, value="2. High requests get 80% of requested amount.")
|
| 1135 |
+
ws_pool.cell(row=6, column=1, value="3. Medium requests get 50% of requested amount.")
|
| 1136 |
+
ws_pool.cell(row=7, column=1, value="4. Low requests get 25% of requested amount.")
|
| 1137 |
+
ws_pool.cell(row=8, column=1, value="5. Process in priority order (Critical first, then High, etc.).")
|
| 1138 |
+
ws_pool.cell(row=9, column=1, value="6. If remaining budget is less than the allocation, give remaining budget.")
|
| 1139 |
+
ws_pool.cell(row=10, column=1, value="7. Once budget is exhausted, remaining requests get $0.")
|
| 1140 |
+
|
| 1141 |
+
# -- Output --
|
| 1142 |
+
ws_out = wb.create_sheet("Output")
|
| 1143 |
+
for c, h in enumerate(["Request ID", "Department", "Priority", "Requested", "Allocation %", "Allocated Amount", "Remaining Budget"], 1):
|
| 1144 |
+
ws_out.cell(row=1, column=c, value=h).font = _bold_font()
|
| 1145 |
+
for i in range(20):
|
| 1146 |
+
r = i + 2
|
| 1147 |
+
ws_out.cell(row=r, column=1, value=requests_data[i][0])
|
| 1148 |
+
for c in range(2, 8):
|
| 1149 |
+
ws_out.cell(row=r, column=c).fill = _yellow_fill()
|
| 1150 |
+
|
| 1151 |
+
wb.save(TEMPLATES_DIR / "budget_allocation.xlsx")
|
| 1152 |
+
|
| 1153 |
+
# Calculate expected allocations
|
| 1154 |
+
priority_rates = {"Critical": 1.0, "High": 0.8, "Medium": 0.5, "Low": 0.25}
|
| 1155 |
+
sorted_requests = sorted(requests_data, key=lambda x: list(priority_rates.keys()).index(x[2]))
|
| 1156 |
+
remaining = total_budget
|
| 1157 |
+
allocations = {}
|
| 1158 |
+
for req_id, dept, pri, amt in sorted_requests:
|
| 1159 |
+
rate = priority_rates[pri]
|
| 1160 |
+
intended = round(amt * rate)
|
| 1161 |
+
actual = min(intended, remaining)
|
| 1162 |
+
remaining -= actual
|
| 1163 |
+
allocations[req_id] = actual
|
| 1164 |
+
|
| 1165 |
+
checks = []
|
| 1166 |
+
for i, (req_id, dept, pri, amt) in enumerate(requests_data):
|
| 1167 |
+
r = i + 2
|
| 1168 |
+
expected = allocations[req_id]
|
| 1169 |
+
checks.append({
|
| 1170 |
+
"sheet": "Output", "cell": f"F{r}",
|
| 1171 |
+
"expected_value_range": [expected * 0.99 - 1, expected * 1.01 + 1],
|
| 1172 |
+
})
|
| 1173 |
+
checks.append({"sheet": "Output", "range": "B2:G21", "check": "no_blanks"})
|
| 1174 |
+
|
| 1175 |
+
_write_json(SCENARIOS_DIR / "conditional_aggregation_02.json", {
|
| 1176 |
+
"id": "conditional_aggregation_02",
|
| 1177 |
+
"description": "Allocate a fixed budget across 20 requests with priority-based allocation rates. Process in priority order; when budget runs out, remaining requests get $0.",
|
| 1178 |
+
"instructions": "Fill the Output sheet by allocating the budget from Budget_Pool across all 20 requests. Read the allocation rules carefully from the Budget_Pool sheet. Process requests in priority order (Critical first, then High, Medium, Low). Within the same priority, process in order of appearance. Each priority level gets a different % of requested amount. Track remaining budget — when it's exhausted, remaining requests get $0.",
|
| 1179 |
+
"workbook": "budget_allocation.xlsx",
|
| 1180 |
+
"max_steps": 55,
|
| 1181 |
+
"category": "conditional_aggregation",
|
| 1182 |
+
})
|
| 1183 |
+
|
| 1184 |
+
_write_json(HIDDEN_TESTS_DIR / "conditional_aggregation_02.json", {
|
| 1185 |
+
"scenario_id": "conditional_aggregation_02",
|
| 1186 |
+
"checks": checks,
|
| 1187 |
+
"target_regions": [{"sheet": "Output", "range": "B2:G21"}],
|
| 1188 |
+
})
|
| 1189 |
+
|
| 1190 |
+
|
| 1191 |
+
# ══════════════════════════════════════════════════════════════════════
|
| 1192 |
+
# Scenario 12: buggy_template_fix_01 — Quarterly Report Debug
|
| 1193 |
+
# ══════════════════════════════════════════════════════════════════════
|
| 1194 |
+
|
| 1195 |
+
def gen_buggy_template_fix_01():
|
| 1196 |
+
wb = openpyxl.Workbook()
|
| 1197 |
+
|
| 1198 |
+
quarters = ["Q1", "Q2", "Q3", "Q4"]
|
| 1199 |
+
metrics = ["Revenue", "COGS", "Gross Profit", "OpEx", "EBITDA", "Depreciation", "Net Income"]
|
| 1200 |
+
|
| 1201 |
+
# Generate quarterly data
|
| 1202 |
+
q_data = {}
|
| 1203 |
+
for q_name in quarters:
|
| 1204 |
+
ws = wb.active if q_name == "Q1" else wb.create_sheet(q_name)
|
| 1205 |
+
if q_name == "Q1":
|
| 1206 |
+
ws.title = "Q1"
|
| 1207 |
+
|
| 1208 |
+
ws.cell(row=1, column=1, value=f"{q_name} Financial Data").font = Font(bold=True, size=12)
|
| 1209 |
+
ws.cell(row=3, column=1, value="Metric").font = _bold_font()
|
| 1210 |
+
ws.cell(row=3, column=2, value="Amount").font = _bold_font()
|
| 1211 |
+
|
| 1212 |
+
revenue = random.randint(800, 1200) * 1000
|
| 1213 |
+
cogs = round(revenue * random.uniform(0.35, 0.45))
|
| 1214 |
+
gross = revenue - cogs
|
| 1215 |
+
opex = round(revenue * random.uniform(0.15, 0.25))
|
| 1216 |
+
ebitda = gross - opex
|
| 1217 |
+
depreciation = round(revenue * 0.05)
|
| 1218 |
+
net_income = ebitda - depreciation
|
| 1219 |
+
|
| 1220 |
+
values = [revenue, cogs, gross, opex, ebitda, depreciation, net_income]
|
| 1221 |
+
q_data[q_name] = dict(zip(metrics, values))
|
| 1222 |
+
|
| 1223 |
+
for i, (metric, val) in enumerate(zip(metrics, values)):
|
| 1224 |
+
ws.cell(row=i + 4, column=1, value=metric)
|
| 1225 |
+
if metric in ("Gross Profit", "EBITDA", "Net Income"):
|
| 1226 |
+
# These should be formulas
|
| 1227 |
+
if metric == "Gross Profit":
|
| 1228 |
+
ws.cell(row=i + 4, column=2, value=f"=B4-B5")
|
| 1229 |
+
elif metric == "EBITDA":
|
| 1230 |
+
ws.cell(row=i + 4, column=2, value=f"=B6-B7")
|
| 1231 |
+
elif metric == "Net Income":
|
| 1232 |
+
ws.cell(row=i + 4, column=2, value=f"=B8-B9")
|
| 1233 |
+
else:
|
| 1234 |
+
ws.cell(row=i + 4, column=2, value=val)
|
| 1235 |
+
|
| 1236 |
+
# -- Annual Summary (with BUGS) --
|
| 1237 |
+
ws_annual = wb.create_sheet("Annual_Summary")
|
| 1238 |
+
ws_annual.cell(row=1, column=1, value="Annual Financial Summary").font = Font(bold=True, size=14)
|
| 1239 |
+
|
| 1240 |
+
ws_annual.cell(row=3, column=1, value="Metric").font = _bold_font()
|
| 1241 |
+
for i, q in enumerate(quarters):
|
| 1242 |
+
ws_annual.cell(row=3, column=i + 2, value=q).font = _bold_font()
|
| 1243 |
+
ws_annual.cell(row=3, column=6, value="Annual Total").font = _bold_font()
|
| 1244 |
+
|
| 1245 |
+
for mi, metric in enumerate(metrics):
|
| 1246 |
+
r = mi + 4
|
| 1247 |
+
ws_annual.cell(row=r, column=1, value=metric)
|
| 1248 |
+
|
| 1249 |
+
for qi, q in enumerate(quarters):
|
| 1250 |
+
col = qi + 2
|
| 1251 |
+
data_row = mi + 4
|
| 1252 |
+
# BUG 1: Q3 references Q2 sheet instead of Q3
|
| 1253 |
+
if qi == 2:
|
| 1254 |
+
ws_annual.cell(row=r, column=col, value=f"=Q2!B{data_row}")
|
| 1255 |
+
# BUG 2: Q4 has off-by-one (references row+1)
|
| 1256 |
+
elif qi == 3:
|
| 1257 |
+
ws_annual.cell(row=r, column=col, value=f"=Q4!B{data_row + 1}")
|
| 1258 |
+
else:
|
| 1259 |
+
ws_annual.cell(row=r, column=col, value=f"={q}!B{data_row}")
|
| 1260 |
+
|
| 1261 |
+
# BUG 3: Annual total only sums Q1 and Q2 (missing Q3 and Q4)
|
| 1262 |
+
ws_annual.cell(row=r, column=6, value=f"=B{r}+C{r}")
|
| 1263 |
+
|
| 1264 |
+
wb.save(TEMPLATES_DIR / "quarterly_report_debug.xlsx")
|
| 1265 |
+
|
| 1266 |
+
# Build expected formulas
|
| 1267 |
+
checks = []
|
| 1268 |
+
for mi, metric in enumerate(metrics):
|
| 1269 |
+
r = mi + 4
|
| 1270 |
+
data_row = mi + 4
|
| 1271 |
+
# Q3 should reference Q3, not Q2
|
| 1272 |
+
checks.append({
|
| 1273 |
+
"sheet": "Annual_Summary", "cell": f"D{r}",
|
| 1274 |
+
"expected_formula": f"=Q3!B{data_row}",
|
| 1275 |
+
})
|
| 1276 |
+
# Q4 should reference correct row
|
| 1277 |
+
checks.append({
|
| 1278 |
+
"sheet": "Annual_Summary", "cell": f"E{r}",
|
| 1279 |
+
"expected_formula": f"=Q4!B{data_row}",
|
| 1280 |
+
})
|
| 1281 |
+
# Annual total should sum all 4 quarters
|
| 1282 |
+
checks.append({
|
| 1283 |
+
"sheet": "Annual_Summary", "cell": f"F{r}",
|
| 1284 |
+
"expected_formula": f"=B{r}+C{r}+D{r}+E{r}",
|
| 1285 |
+
})
|
| 1286 |
+
|
| 1287 |
+
_write_json(SCENARIOS_DIR / "buggy_template_fix_01.json", {
|
| 1288 |
+
"id": "buggy_template_fix_01",
|
| 1289 |
+
"description": "Debug a quarterly financial report template. The Annual_Summary sheet has three types of formula bugs: Q3 references Q2 data, Q4 has off-by-one row errors, and Annual Totals only sum 2 of 4 quarters.",
|
| 1290 |
+
"instructions": "The Annual_Summary sheet should show each metric's value for Q1-Q4 (pulled from the individual quarter sheets) and an Annual Total. Three bugs exist: (1) The Q3 column references the Q2 sheet instead of Q3. (2) The Q4 column references the wrong row (off by one). (3) The Annual Total formula only sums Q1+Q2 instead of all four quarters. Fix all formulas in the Annual_Summary sheet. Do NOT modify the individual quarter sheets.",
|
| 1291 |
+
"workbook": "quarterly_report_debug.xlsx",
|
| 1292 |
+
"max_steps": 50,
|
| 1293 |
+
"category": "buggy_template_fix",
|
| 1294 |
+
})
|
| 1295 |
+
|
| 1296 |
+
_write_json(HIDDEN_TESTS_DIR / "buggy_template_fix_01.json", {
|
| 1297 |
+
"scenario_id": "buggy_template_fix_01",
|
| 1298 |
+
"checks": checks,
|
| 1299 |
+
"target_regions": [{"sheet": "Annual_Summary", "range": "D4:F10"}],
|
| 1300 |
+
})
|
| 1301 |
+
|
| 1302 |
+
|
| 1303 |
+
# ══════════════════════════════════════════════════════════════════════
|
| 1304 |
+
# Main
|
| 1305 |
+
# ══════════════════════════════════════════════════════════════════════
|
| 1306 |
+
|
| 1307 |
+
def main():
|
| 1308 |
+
generators = [
|
| 1309 |
+
gen_formula_repair_01,
|
| 1310 |
+
gen_formula_repair_02,
|
| 1311 |
+
gen_cross_sheet_lookup_01,
|
| 1312 |
+
gen_cross_sheet_lookup_02,
|
| 1313 |
+
gen_messy_table_extraction_01,
|
| 1314 |
+
gen_schedule_grid_fill_01,
|
| 1315 |
+
gen_ledger_reconciliation_01,
|
| 1316 |
+
gen_ledger_reconciliation_02,
|
| 1317 |
+
gen_range_transformation_01,
|
| 1318 |
+
gen_conditional_aggregation_01,
|
| 1319 |
+
gen_conditional_aggregation_02,
|
| 1320 |
+
gen_buggy_template_fix_01,
|
| 1321 |
+
]
|
| 1322 |
+
|
| 1323 |
+
for gen_fn in generators:
|
| 1324 |
+
name = gen_fn.__name__.replace("gen_", "")
|
| 1325 |
+
print(f"Generating {name}...")
|
| 1326 |
+
gen_fn()
|
| 1327 |
+
print(f" ✓ {name}")
|
| 1328 |
+
|
| 1329 |
+
print(f"\nGenerated {len(generators)} scenarios:")
|
| 1330 |
+
print(f" Templates: {TEMPLATES_DIR}")
|
| 1331 |
+
print(f" Scenarios: {SCENARIOS_DIR}")
|
| 1332 |
+
print(f" Hidden Tests: {HIDDEN_TESTS_DIR}")
|
| 1333 |
+
|
| 1334 |
+
|
| 1335 |
+
if __name__ == "__main__":
|
| 1336 |
+
main()
|
models.py
ADDED
|
@@ -0,0 +1,86 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Data models for the Spreadsheet Environment.
|
| 2 |
+
|
| 3 |
+
SpreadsheetAction has explicit Pydantic fields for MCP-style tool calls
|
| 4 |
+
(tool_name, arguments_json) compatible with the OpenEnv web interface.
|
| 5 |
+
"""
|
| 6 |
+
|
| 7 |
+
from __future__ import annotations
|
| 8 |
+
|
| 9 |
+
import json as _json
|
| 10 |
+
from typing import Any, Union
|
| 11 |
+
|
| 12 |
+
from pydantic import ConfigDict, Field, TypeAdapter, model_validator
|
| 13 |
+
|
| 14 |
+
from openenv.core.env_server.mcp_types import (
|
| 15 |
+
CallToolAction,
|
| 16 |
+
CallToolObservation,
|
| 17 |
+
ListToolsAction,
|
| 18 |
+
ListToolsObservation,
|
| 19 |
+
)
|
| 20 |
+
from openenv.core.env_server.types import Action, Observation, State
|
| 21 |
+
|
| 22 |
+
_mcp_action_adapter = TypeAdapter(Union[ListToolsAction, CallToolAction])
|
| 23 |
+
|
| 24 |
+
_AVAILABLE_TOOLS = (
|
| 25 |
+
"list_tools, get_session_info, list_scenarios, load_scenario, "
|
| 26 |
+
"list_sheets, read_range, write_cell, write_range, inspect_formula, "
|
| 27 |
+
"list_named_targets, validate_partial, submit_workbook, "
|
| 28 |
+
"get_edit_history, reset_scenario"
|
| 29 |
+
)
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
class SpreadsheetAction(Action):
|
| 33 |
+
"""Action with explicit fields for the web UI and MCP compatibility."""
|
| 34 |
+
|
| 35 |
+
model_config = ConfigDict(
|
| 36 |
+
extra="forbid",
|
| 37 |
+
validate_assignment=True,
|
| 38 |
+
arbitrary_types_allowed=True,
|
| 39 |
+
)
|
| 40 |
+
|
| 41 |
+
tool_name: str = Field(
|
| 42 |
+
default="list_tools",
|
| 43 |
+
description=f"MCP tool to invoke. Available: {_AVAILABLE_TOOLS}",
|
| 44 |
+
)
|
| 45 |
+
arguments_json: str = Field(
|
| 46 |
+
default="{}",
|
| 47 |
+
description=(
|
| 48 |
+
'Tool arguments as a JSON string. Examples: '
|
| 49 |
+
'"{}" for no args, '
|
| 50 |
+
'\'{"scenario_id":"formula_repair_01"}\' for load_scenario, '
|
| 51 |
+
'\'{"sheet":"Summary","range":"A1:D10"}\' for read_range, '
|
| 52 |
+
'\'{"sheet":"Summary","cell":"C15","value":"=SUM(A1:A10)"}\' for write_cell'
|
| 53 |
+
),
|
| 54 |
+
)
|
| 55 |
+
|
| 56 |
+
@model_validator(mode="after")
|
| 57 |
+
def _validate_json(self) -> "SpreadsheetAction":
|
| 58 |
+
if self.arguments_json.strip():
|
| 59 |
+
_json.loads(self.arguments_json)
|
| 60 |
+
return self
|
| 61 |
+
|
| 62 |
+
@classmethod
|
| 63 |
+
def model_validate(cls, data: Any, **kwargs: Any) -> Action:
|
| 64 |
+
if isinstance(data, dict) and data.get("type") in ("call_tool", "list_tools"):
|
| 65 |
+
return _mcp_action_adapter.validate_python(data)
|
| 66 |
+
return super().model_validate(data, **kwargs)
|
| 67 |
+
|
| 68 |
+
def to_mcp_action(self) -> Action:
|
| 69 |
+
if self.tool_name == "list_tools":
|
| 70 |
+
return ListToolsAction()
|
| 71 |
+
args = _json.loads(self.arguments_json) if self.arguments_json else {}
|
| 72 |
+
return CallToolAction(tool_name=self.tool_name, arguments=args)
|
| 73 |
+
|
| 74 |
+
|
| 75 |
+
SpreadsheetObservation = CallToolObservation
|
| 76 |
+
SpreadsheetState = State
|
| 77 |
+
|
| 78 |
+
__all__ = [
|
| 79 |
+
"SpreadsheetAction",
|
| 80 |
+
"SpreadsheetObservation",
|
| 81 |
+
"SpreadsheetState",
|
| 82 |
+
"CallToolAction",
|
| 83 |
+
"CallToolObservation",
|
| 84 |
+
"ListToolsAction",
|
| 85 |
+
"ListToolsObservation",
|
| 86 |
+
]
|
openenv.yaml
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
spec_version: 1
|
| 2 |
+
name: spreadsheet
|
| 3 |
+
description: "Spreadsheet — exact workbook manipulation and reasoning over realistic spreadsheet tasks"
|
| 4 |
+
type: space
|
| 5 |
+
runtime: fastapi
|
| 6 |
+
app: server.app:app
|
| 7 |
+
port: 8000
|
| 8 |
+
|
openenv_spreadsheet.egg-info/PKG-INFO
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
Metadata-Version: 2.4
|
| 2 |
+
Name: openenv-spreadsheet
|
| 3 |
+
Version: 0.1.0
|
| 4 |
+
Summary: Spreadsheet gym — exact workbook manipulation and reasoning over realistic spreadsheet tasks
|
| 5 |
+
Requires-Python: >=3.11
|
| 6 |
+
Requires-Dist: openenv-core @ git+https://github.com/meta-pytorch/OpenEnv.git@v0.2.1
|
| 7 |
+
Requires-Dist: fastapi>=0.115.0
|
| 8 |
+
Requires-Dist: pydantic>=2.0.0
|
| 9 |
+
Requires-Dist: uvicorn[standard]>=0.24.0
|
| 10 |
+
Requires-Dist: fastmcp>=0.1.0
|
| 11 |
+
Requires-Dist: httpx>=0.25.0
|
| 12 |
+
Requires-Dist: openpyxl>=3.1.0
|
| 13 |
+
Requires-Dist: pandas>=2.0.0
|
| 14 |
+
Requires-Dist: formulas>=1.2.0
|
| 15 |
+
Provides-Extra: dev
|
| 16 |
+
Requires-Dist: pytest>=8.0.0; extra == "dev"
|
| 17 |
+
Requires-Dist: pytest-cov>=4.0.0; extra == "dev"
|
openenv_spreadsheet.egg-info/SOURCES.txt
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
README.md
|
| 2 |
+
__init__.py
|
| 3 |
+
client.py
|
| 4 |
+
generate_scenarios.py
|
| 5 |
+
models.py
|
| 6 |
+
openenv.yaml
|
| 7 |
+
pyproject.toml
|
| 8 |
+
./__init__.py
|
| 9 |
+
./client.py
|
| 10 |
+
./generate_scenarios.py
|
| 11 |
+
./models.py
|
| 12 |
+
./openenv.yaml
|
| 13 |
+
openenv_spreadsheet.egg-info/PKG-INFO
|
| 14 |
+
openenv_spreadsheet.egg-info/SOURCES.txt
|
| 15 |
+
openenv_spreadsheet.egg-info/dependency_links.txt
|
| 16 |
+
openenv_spreadsheet.egg-info/entry_points.txt
|
| 17 |
+
openenv_spreadsheet.egg-info/requires.txt
|
| 18 |
+
openenv_spreadsheet.egg-info/top_level.txt
|
| 19 |
+
server/__init__.py
|
| 20 |
+
server/app.py
|
| 21 |
+
server/formula_utils.py
|
| 22 |
+
server/scenario_loader.py
|
| 23 |
+
server/spreadsheet_environment.py
|
| 24 |
+
server/workbook_engine.py
|
openenv_spreadsheet.egg-info/dependency_links.txt
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
|
openenv_spreadsheet.egg-info/entry_points.txt
ADDED
|
@@ -0,0 +1,2 @@
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[console_scripts]
|
| 2 |
+
server = spreadsheet.server.app:main
|
openenv_spreadsheet.egg-info/requires.txt
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
openenv-core @ git+https://github.com/meta-pytorch/OpenEnv.git@v0.2.1
|
| 2 |
+
fastapi>=0.115.0
|
| 3 |
+
pydantic>=2.0.0
|
| 4 |
+
uvicorn[standard]>=0.24.0
|
| 5 |
+
fastmcp>=0.1.0
|
| 6 |
+
httpx>=0.25.0
|
| 7 |
+
openpyxl>=3.1.0
|
| 8 |
+
pandas>=2.0.0
|
| 9 |
+
formulas>=1.2.0
|
| 10 |
+
|
| 11 |
+
[dev]
|
| 12 |
+
pytest>=8.0.0
|
| 13 |
+
pytest-cov>=4.0.0
|
openenv_spreadsheet.egg-info/top_level.txt
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
spreadsheet
|
pyproject.toml
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[build-system]
|
| 2 |
+
requires = ["setuptools>=45", "wheel"]
|
| 3 |
+
build-backend = "setuptools.build_meta"
|
| 4 |
+
|
| 5 |
+
[project]
|
| 6 |
+
name = "openenv-spreadsheet"
|
| 7 |
+
version = "0.1.0"
|
| 8 |
+
description = "Spreadsheet gym — exact workbook manipulation and reasoning over realistic spreadsheet tasks"
|
| 9 |
+
requires-python = ">=3.11"
|
| 10 |
+
dependencies = [
|
| 11 |
+
"openenv-core @ git+https://github.com/meta-pytorch/OpenEnv.git@v0.2.1",
|
| 12 |
+
"fastapi>=0.115.0",
|
| 13 |
+
"pydantic>=2.0.0",
|
| 14 |
+
"uvicorn[standard]>=0.24.0",
|
| 15 |
+
"fastmcp>=0.1.0",
|
| 16 |
+
"httpx>=0.25.0",
|
| 17 |
+
"openpyxl>=3.1.0",
|
| 18 |
+
"pandas>=2.0.0",
|
| 19 |
+
"formulas>=1.2.0",
|
| 20 |
+
]
|
| 21 |
+
|
| 22 |
+
[project.optional-dependencies]
|
| 23 |
+
dev = [
|
| 24 |
+
"pytest>=8.0.0",
|
| 25 |
+
"pytest-cov>=4.0.0",
|
| 26 |
+
]
|
| 27 |
+
|
| 28 |
+
[project.scripts]
|
| 29 |
+
server = "spreadsheet.server.app:main"
|
| 30 |
+
|
| 31 |
+
[tool.setuptools]
|
| 32 |
+
include-package-data = true
|
| 33 |
+
packages = ["spreadsheet", "spreadsheet.server"]
|
| 34 |
+
package-dir = { "spreadsheet" = ".", "spreadsheet.server" = "server" }
|
| 35 |
+
|
| 36 |
+
[tool.setuptools.package-data]
|
| 37 |
+
spreadsheet = ["openenv.yaml"]
|
scenarios/.gitkeep
ADDED
|
File without changes
|
scenarios/buggy_template_fix_01.json
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"id": "buggy_template_fix_01",
|
| 3 |
+
"description": "Debug a quarterly financial report template. The Annual_Summary sheet has three types of formula bugs: Q3 references Q2 data, Q4 has off-by-one row errors, and Annual Totals only sum 2 of 4 quarters.",
|
| 4 |
+
"instructions": "The Annual_Summary sheet should show each metric's value for Q1-Q4 (pulled from the individual quarter sheets) and an Annual Total. Three bugs exist: (1) The Q3 column references the Q2 sheet instead of Q3. (2) The Q4 column references the wrong row (off by one). (3) The Annual Total formula only sums Q1+Q2 instead of all four quarters. Fix all formulas in the Annual_Summary sheet. Do NOT modify the individual quarter sheets.",
|
| 5 |
+
"workbook": "quarterly_report_debug.xlsx",
|
| 6 |
+
"max_steps": 50,
|
| 7 |
+
"category": "buggy_template_fix"
|
| 8 |
+
}
|
scenarios/conditional_aggregation_01.json
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"id": "conditional_aggregation_01",
|
| 3 |
+
"description": "Calculate tiered sales commissions for 15 salespeople. Commission rates depend on annual total tier. West region gets a +2% bonus on top of the base tier rate.",
|
| 4 |
+
"instructions": "Fill the Commissions sheet for each salesperson. Look up their Annual Sales from the Sales sheet, determine their tier from Commission_Rules, and calculate the commission. IMPORTANT: Read the Commission_Rules sheet carefully \u2014 the commission rate is applied to the FULL annual total (not marginal). The West region gets an additional +2% regional bonus. Fill all columns: Region, Annual Sales, Tier, Base Rate, Regional Bonus, Total Rate, Commission Amount.",
|
| 5 |
+
"workbook": "sales_commission.xlsx",
|
| 6 |
+
"max_steps": 55,
|
| 7 |
+
"category": "conditional_aggregation"
|
| 8 |
+
}
|
scenarios/conditional_aggregation_02.json
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"id": "conditional_aggregation_02",
|
| 3 |
+
"description": "Allocate a fixed budget across 20 requests with priority-based allocation rates. Process in priority order; when budget runs out, remaining requests get $0.",
|
| 4 |
+
"instructions": "Fill the Output sheet by allocating the budget from Budget_Pool across all 20 requests. Read the allocation rules carefully from the Budget_Pool sheet. Process requests in priority order (Critical first, then High, Medium, Low). Within the same priority, process in order of appearance. Each priority level gets a different % of requested amount. Track remaining budget \u2014 when it's exhausted, remaining requests get $0.",
|
| 5 |
+
"workbook": "budget_allocation.xlsx",
|
| 6 |
+
"max_steps": 55,
|
| 7 |
+
"category": "conditional_aggregation"
|
| 8 |
+
}
|
scenarios/cross_sheet_lookup_01.json
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"id": "cross_sheet_lookup_01",
|
| 3 |
+
"description": "Aggregate product revenue by region and category across two quarterly sales sheets. Some product codes have typos. The Summary sheet must be filled with correct totals.",
|
| 4 |
+
"instructions": "Fill the Summary sheet with revenue totals broken down by Region (rows) and Product Category (columns: Hardware, Services, Software). Data is in Sales_Q1 and Sales_Q2. Use the Products sheet to map product codes to categories. WARNING: Some product codes in the sales sheets have typos (missing dashes or lowercase). You must account for these when aggregating. The Total column should sum across categories for each region. Grand Total row should sum each column.",
|
| 5 |
+
"workbook": "product_revenue_by_region.xlsx",
|
| 6 |
+
"max_steps": 60,
|
| 7 |
+
"category": "cross_sheet_lookup"
|
| 8 |
+
}
|
scenarios/cross_sheet_lookup_02.json
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"id": "cross_sheet_lookup_02",
|
| 3 |
+
"description": "Calculate employee bonuses by cross-referencing Employees, Bonus_Tiers (non-standard layout at column F), and Performance sheets. Fill the Payroll sheet.",
|
| 4 |
+
"instructions": "Fill the Payroll sheet for all 25 employees. For each employee: (1) Look up their Name, Level, and Base Salary from the Employees sheet. (2) Look up their average performance score from the Performance sheet. (3) Find the bonus rate from the Bonus_Tiers sheet (NOTE: the tier table is in columns F-I, not A-D). (4) If the employee's avg score meets the minimum performance threshold for their tier, apply the bonus rate; otherwise bonus is 0. (5) Bonus Amount = Base Salary \u00d7 Bonus Rate. (6) Total Comp = Base Salary + Bonus Amount.",
|
| 5 |
+
"workbook": "employee_bonus_calculation.xlsx",
|
| 6 |
+
"max_steps": 60,
|
| 7 |
+
"category": "cross_sheet_lookup"
|
| 8 |
+
}
|
scenarios/formula_repair_01.json
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"id": "formula_repair_01",
|
| 3 |
+
"description": "Fix broken formulas in a multi-department budget workbook. Summary sheet has wrong ranges and references to a deleted sheet. Marketing total comp formulas reference a non-existent OldBudget sheet.",
|
| 4 |
+
"instructions": "The Summary sheet has broken formulas. Engineering total compensation references wrong cell ranges. Marketing total compensation references a deleted 'OldBudget' sheet. Fix all broken formulas so Summary correctly aggregates total compensation from Engineering and Marketing sheets. Also fix the Marketing sheet's Total Comp column to use the correct formula (Base Salary \u00d7 (1 + Bonus %)). Check the HR Policies sheet for the correct bonus calculation method. There is a hidden Metadata sheet with hints.",
|
| 5 |
+
"workbook": "multi_department_budget.xlsx",
|
| 6 |
+
"max_steps": 50,
|
| 7 |
+
"category": "formula_repair"
|
| 8 |
+
}
|
scenarios/formula_repair_02.json
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"id": "formula_repair_02",
|
| 3 |
+
"description": "Fix cascading formula errors in a 5-year financial projection. Revenue growth, tax rates, and discount factors reference wrong cells or use hardcoded values instead of the Assumptions sheet.",
|
| 4 |
+
"instructions": "This workbook has a 5-year financial projection with three sheets: Assumptions, Revenue, and DCF. Multiple formulas contain errors: (1) Revenue years 4-5 use hardcoded 5% growth instead of the Assumptions growth rate. (2) Tax calculations use 25% instead of the Assumptions tax rate (21%). (3) The Assumptions sheet has the correct values \u2014 all formulas should reference it. Fix all broken formulas in Revenue and DCF sheets to properly reference Assumptions.",
|
| 5 |
+
"workbook": "cascading_formula_errors.xlsx",
|
| 6 |
+
"max_steps": 50,
|
| 7 |
+
"category": "formula_repair"
|
| 8 |
+
}
|
scenarios/ledger_reconciliation_01.json
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"id": "ledger_reconciliation_01",
|
| 3 |
+
"description": "Reconcile a bank statement against an internal ledger. Find mismatches, missing entries, and amount discrepancies. Fill the Reconciled sheet.",
|
| 4 |
+
"instructions": "Compare Bank_Statement and Internal_Ledger to produce a reconciliation report in the Reconciled sheet. For each transaction: match by date and description. Record the Bank Amount, Ledger Amount, Difference (Bank - Ledger), and Status (Matched/Mismatch/Bank Only/Ledger Only). Include ALL transactions from both sources. Sort by date.",
|
| 5 |
+
"workbook": "bank_reconciliation.xlsx",
|
| 6 |
+
"max_steps": 60,
|
| 7 |
+
"category": "ledger_reconciliation"
|
| 8 |
+
}
|
scenarios/ledger_reconciliation_02.json
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"id": "ledger_reconciliation_02",
|
| 3 |
+
"description": "Reconcile USD and EUR transaction sheets into a unified summary. EUR dates are in DD-MM-YYYY format, USD dates in MM/DD/YYYY. Convert EUR to USD using the Exchange_Rates sheet.",
|
| 4 |
+
"instructions": "The workbook has USD and EUR transaction sheets with different date formats. Convert all EUR transactions to USD using the monthly exchange rate from the Exchange_Rates sheet (match each transaction's month to the correct rate). Fill the Summary sheet with: Total USD Transactions, Total EUR Transactions converted to USD, Grand Total, and transaction counts. Dates in the transaction sheets use different formats \u2014 be careful when determining which month each EUR transaction falls in.",
|
| 5 |
+
"workbook": "multi_currency_reconciliation.xlsx",
|
| 6 |
+
"max_steps": 55,
|
| 7 |
+
"category": "ledger_reconciliation"
|
| 8 |
+
}
|
scenarios/messy_table_extraction_01.json
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"id": "messy_table_extraction_01",
|
| 3 |
+
"description": "Extract and clean invoice data from a messy raw export with mixed date formats, section headers mixed in with data rows, and blank separator rows. All dates must be normalized to ISO format.",
|
| 4 |
+
"instructions": "The Raw_Invoices sheet has messy data exported from a legacy system: title rows at top, section header rows (like '--- Q1 2024 ---') mixed in with data, blank separator rows, and inconsistent date formats (MM/DD/YYYY, DD-MM-YYYY, and ISO). Extract all actual invoice rows into the Processed sheet with: (1) Invoice #, (2) Date in ISO format (YYYY-MM-DD), (3) Vendor, (4) Amount, (5) Status. Skip section headers and blank rows. Dates must all be converted to ISO format.",
|
| 5 |
+
"workbook": "vendor_invoice_processing.xlsx",
|
| 6 |
+
"max_steps": 60,
|
| 7 |
+
"category": "messy_table_extraction"
|
| 8 |
+
}
|
scenarios/range_transformation_01.json
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"id": "range_transformation_01",
|
| 3 |
+
"description": "Pivot long-format employee metrics data into a wide-format table. Each employee gets one row with Sales, Returns, and Net Revenue for each of 6 months, plus a total.",
|
| 4 |
+
"instructions": "The Raw_Data sheet has employee performance metrics in long format (one row per employee-month-metric combination). Pivot this into the Pivot_Output sheet: one row per employee (sorted alphabetically), with columns for each month's Sales, Returns, and Net Revenue. Add a Total Net Revenue column at the end. The headers are pre-filled; fill the data cells.",
|
| 5 |
+
"workbook": "data_pivot_reshape.xlsx",
|
| 6 |
+
"max_steps": 60,
|
| 7 |
+
"category": "range_transformation"
|
| 8 |
+
}
|
scenarios/schedule_grid_fill_01.json
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"id": "schedule_grid_fill_01",
|
| 3 |
+
"description": "Fill an employee schedule grid for 12 employees across 7 days, respecting prose constraints on max days, shift transitions, minimum coverage, and availability exceptions.",
|
| 4 |
+
"instructions": "Fill the Output sheet with shift codes (M=Morning, A=Afternoon, N=Night, X=Off) for each employee and day. You must satisfy ALL constraints from the Constraints sheet and respect the availability exceptions from the Availability sheet. Unavailable employees must have X for that day. Check the Shift_Codes sheet for valid codes.",
|
| 5 |
+
"workbook": "employee_schedule_grid.xlsx",
|
| 6 |
+
"max_steps": 70,
|
| 7 |
+
"category": "schedule_grid_fill"
|
| 8 |
+
}
|
server/__init__.py
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Meta Platforms, Inc. and affiliates.
|
| 2 |
+
# All rights reserved.
|
| 3 |
+
#
|
| 4 |
+
# This source code is licensed under the BSD-style license found in the
|
| 5 |
+
# LICENSE file in the root directory of this source tree.
|
| 6 |
+
|
| 7 |
+
"""Spreadsheet environment server components."""
|
| 8 |
+
|
| 9 |
+
from .spreadsheet_environment import SpreadsheetEnvironment
|
| 10 |
+
|
| 11 |
+
__all__ = ["SpreadsheetEnvironment"]
|
server/app.py
ADDED
|
@@ -0,0 +1,41 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""FastAPI application for the Spreadsheet Environment."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
import os
|
| 6 |
+
import sys
|
| 7 |
+
from pathlib import Path
|
| 8 |
+
|
| 9 |
+
try:
|
| 10 |
+
from openenv.core.env_server.http_server import create_app
|
| 11 |
+
except ImportError as e:
|
| 12 |
+
raise ImportError(
|
| 13 |
+
"openenv is required. Install with: uv sync"
|
| 14 |
+
) from e
|
| 15 |
+
|
| 16 |
+
try:
|
| 17 |
+
from spreadsheet.models import SpreadsheetAction, SpreadsheetObservation
|
| 18 |
+
from spreadsheet.server.spreadsheet_environment import SpreadsheetEnvironment
|
| 19 |
+
except ImportError:
|
| 20 |
+
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
| 21 |
+
from models import SpreadsheetAction, SpreadsheetObservation
|
| 22 |
+
from server.spreadsheet_environment import SpreadsheetEnvironment
|
| 23 |
+
|
| 24 |
+
MAX_CONCURRENT_ENVS = int(os.getenv("MAX_CONCURRENT_ENVS", "8"))
|
| 25 |
+
|
| 26 |
+
app = create_app(
|
| 27 |
+
SpreadsheetEnvironment,
|
| 28 |
+
SpreadsheetAction,
|
| 29 |
+
SpreadsheetObservation,
|
| 30 |
+
env_name="spreadsheet",
|
| 31 |
+
max_concurrent_envs=MAX_CONCURRENT_ENVS,
|
| 32 |
+
)
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
def main(host: str = "0.0.0.0", port: int = 8000):
|
| 36 |
+
import uvicorn
|
| 37 |
+
uvicorn.run(app, host=host, port=port)
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
if __name__ == "__main__":
|
| 41 |
+
main()
|
server/formula_utils.py
ADDED
|
@@ -0,0 +1,39 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Formula utilities — Excel-compatible formula evaluation using the formulas library."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
from typing import Any, Optional
|
| 6 |
+
|
| 7 |
+
import openpyxl
|
| 8 |
+
|
| 9 |
+
|
| 10 |
+
def evaluate_formula(wb: openpyxl.Workbook, sheet_name: str, cell_ref: str) -> Optional[Any]:
|
| 11 |
+
"""Evaluate an Excel formula in-memory using the formulas library.
|
| 12 |
+
|
| 13 |
+
Falls back to openpyxl data_only reload if formulas library fails.
|
| 14 |
+
Returns the computed value or None on failure.
|
| 15 |
+
"""
|
| 16 |
+
try:
|
| 17 |
+
import formulas
|
| 18 |
+
except ImportError:
|
| 19 |
+
return _fallback_evaluate(wb, sheet_name, cell_ref)
|
| 20 |
+
|
| 21 |
+
try:
|
| 22 |
+
xl_model = formulas.ExcelModel().loads(wb.path).finish()
|
| 23 |
+
solution = xl_model.calculate()
|
| 24 |
+
key = f"'{sheet_name}'!{cell_ref.upper()}"
|
| 25 |
+
return solution.get(key)
|
| 26 |
+
except Exception:
|
| 27 |
+
return _fallback_evaluate(wb, sheet_name, cell_ref)
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
def _fallback_evaluate(wb: openpyxl.Workbook, sheet_name: str, cell_ref: str) -> Optional[Any]:
|
| 31 |
+
"""Fallback: reload workbook with data_only=True to get cached values."""
|
| 32 |
+
if not wb.path:
|
| 33 |
+
return None
|
| 34 |
+
try:
|
| 35 |
+
wb_data = openpyxl.load_workbook(wb.path, data_only=True)
|
| 36 |
+
ws = wb_data[sheet_name]
|
| 37 |
+
return ws[cell_ref].value
|
| 38 |
+
except Exception:
|
| 39 |
+
return None
|
server/scenario_loader.py
ADDED
|
@@ -0,0 +1,62 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Scenario loader — load scenario definitions from JSON files."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
import json
|
| 6 |
+
import os
|
| 7 |
+
import shutil
|
| 8 |
+
from pathlib import Path
|
| 9 |
+
from typing import Optional
|
| 10 |
+
|
| 11 |
+
WORKBOOKS_DIR = os.getenv("WORKBOOKS_DIR", str(Path(__file__).resolve().parent.parent / "workbooks"))
|
| 12 |
+
SCENARIOS_DIR = os.getenv("SCENARIOS_DIR", str(Path(__file__).resolve().parent.parent / "scenarios"))
|
| 13 |
+
FIXTURES_DIR = os.path.join(WORKBOOKS_DIR, "fixtures")
|
| 14 |
+
TEMPLATES_DIR = os.path.join(WORKBOOKS_DIR, "templates")
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
def list_scenarios() -> list[dict]:
|
| 18 |
+
"""List all available scenario definitions."""
|
| 19 |
+
if not os.path.isdir(SCENARIOS_DIR):
|
| 20 |
+
return []
|
| 21 |
+
scenarios = []
|
| 22 |
+
for f in sorted(os.listdir(SCENARIOS_DIR)):
|
| 23 |
+
if not f.endswith(".json"):
|
| 24 |
+
continue
|
| 25 |
+
try:
|
| 26 |
+
data = load_scenario_def(f.replace(".json", ""))
|
| 27 |
+
scenarios.append({
|
| 28 |
+
"scenario_id": data.get("id", f.replace(".json", "")),
|
| 29 |
+
"description": data.get("description", ""),
|
| 30 |
+
"workbook": data.get("workbook", ""),
|
| 31 |
+
"max_steps": data.get("max_steps", 50),
|
| 32 |
+
})
|
| 33 |
+
except Exception:
|
| 34 |
+
continue
|
| 35 |
+
return scenarios
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
def load_scenario_def(scenario_id: str) -> dict:
|
| 39 |
+
"""Load a single scenario definition JSON."""
|
| 40 |
+
path = os.path.join(SCENARIOS_DIR, f"{scenario_id}.json")
|
| 41 |
+
if not os.path.isfile(path):
|
| 42 |
+
raise FileNotFoundError(f"Scenario '{scenario_id}' not found at {path}")
|
| 43 |
+
with open(path) as f:
|
| 44 |
+
return json.load(f)
|
| 45 |
+
|
| 46 |
+
|
| 47 |
+
def prepare_workbook_for_session(scenario_id: str, session_id: str) -> str:
|
| 48 |
+
"""Copy the template workbook to a session-specific fixture path.
|
| 49 |
+
|
| 50 |
+
Returns the path to the session's workbook copy.
|
| 51 |
+
"""
|
| 52 |
+
scenario = load_scenario_def(scenario_id)
|
| 53 |
+
template_name = scenario.get("workbook", f"{scenario_id}.xlsx")
|
| 54 |
+
template_path = os.path.join(TEMPLATES_DIR, template_name)
|
| 55 |
+
|
| 56 |
+
if not os.path.isfile(template_path):
|
| 57 |
+
raise FileNotFoundError(f"Template workbook not found: {template_path}")
|
| 58 |
+
|
| 59 |
+
os.makedirs(FIXTURES_DIR, exist_ok=True)
|
| 60 |
+
session_wb_path = os.path.join(FIXTURES_DIR, f"{session_id}_{template_name}")
|
| 61 |
+
shutil.copy2(template_path, session_wb_path)
|
| 62 |
+
return session_wb_path
|
server/spreadsheet_environment.py
ADDED
|
@@ -0,0 +1,445 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Spreadsheet Environment — MCPEnvironment with 13 real MCP tools."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
import json
|
| 6 |
+
import os
|
| 7 |
+
from typing import Any, Optional
|
| 8 |
+
from uuid import uuid4
|
| 9 |
+
|
| 10 |
+
from fastmcp import FastMCP
|
| 11 |
+
|
| 12 |
+
from openenv.core.env_server.mcp_environment import MCPEnvironment
|
| 13 |
+
from openenv.core.env_server.types import Action, EnvironmentMetadata, Observation, State
|
| 14 |
+
|
| 15 |
+
from .scenario_loader import list_scenarios as _list_scenarios
|
| 16 |
+
from .scenario_loader import load_scenario_def, prepare_workbook_for_session
|
| 17 |
+
from .workbook_engine import WorkbookEngine, WorkbookSession
|
| 18 |
+
|
| 19 |
+
WRITE_TOOLS = frozenset({"write_cell", "write_range"})
|
| 20 |
+
READ_TOOLS = frozenset({"read_range", "read_cell"})
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
class SpreadsheetEnvironment(MCPEnvironment):
|
| 24 |
+
"""Workbook manipulation environment — 13 MCP tools exposed via OpenEnv."""
|
| 25 |
+
|
| 26 |
+
SUPPORTS_CONCURRENT_SESSIONS = True
|
| 27 |
+
|
| 28 |
+
def __init__(self):
|
| 29 |
+
mcp = FastMCP("spreadsheet")
|
| 30 |
+
self._session_id: Optional[str] = None
|
| 31 |
+
self._state = State(episode_id=str(uuid4()), step_count=0)
|
| 32 |
+
self._action_history: list[dict] = []
|
| 33 |
+
self._engine = WorkbookEngine()
|
| 34 |
+
self._scenario: Optional[dict] = None
|
| 35 |
+
self._last_validate_passed: int = 0
|
| 36 |
+
|
| 37 |
+
def _record(tool_name: str, **kwargs: Any) -> None:
|
| 38 |
+
self._action_history.append({"tool": tool_name, "arguments": kwargs})
|
| 39 |
+
|
| 40 |
+
# ── Tool 1: get_session_info ──────────────────────────────────
|
| 41 |
+
|
| 42 |
+
@mcp.tool()
|
| 43 |
+
def get_session_info() -> dict:
|
| 44 |
+
"""Return current session metadata: session ID, loaded scenario, step count, edit count, and solve status."""
|
| 45 |
+
_record("get_session_info")
|
| 46 |
+
if not self._session_id:
|
| 47 |
+
return {"status": "no_session", "message": "Reset the environment first."}
|
| 48 |
+
return self._engine.get_session_info(self._session_id)
|
| 49 |
+
|
| 50 |
+
# ── Tool 2: list_scenarios ────────────────────────────────────
|
| 51 |
+
|
| 52 |
+
@mcp.tool()
|
| 53 |
+
def list_scenarios() -> dict:
|
| 54 |
+
"""List all available spreadsheet task scenarios. Each entry has a scenario_id, description, workbook name, and max_steps."""
|
| 55 |
+
_record("list_scenarios")
|
| 56 |
+
scenarios = _list_scenarios()
|
| 57 |
+
return {"scenarios": scenarios, "count": len(scenarios)}
|
| 58 |
+
|
| 59 |
+
# ── Tool 3: load_scenario ─────────────────────────────────────
|
| 60 |
+
|
| 61 |
+
@mcp.tool()
|
| 62 |
+
def load_scenario(scenario_id: str) -> dict:
|
| 63 |
+
"""Load a scenario and its workbook to begin working on a task.
|
| 64 |
+
|
| 65 |
+
Args:
|
| 66 |
+
scenario_id: The ID of the scenario to load (from list_scenarios).
|
| 67 |
+
|
| 68 |
+
Returns the scenario description, instructions, sheet list, and target regions.
|
| 69 |
+
"""
|
| 70 |
+
_record("load_scenario", scenario_id=scenario_id)
|
| 71 |
+
try:
|
| 72 |
+
scenario_def = load_scenario_def(scenario_id)
|
| 73 |
+
except FileNotFoundError as e:
|
| 74 |
+
return {"error": str(e)}
|
| 75 |
+
|
| 76 |
+
wb_path = prepare_workbook_for_session(scenario_id, self._session_id)
|
| 77 |
+
session = WorkbookSession(
|
| 78 |
+
session_id=self._session_id,
|
| 79 |
+
scenario_id=scenario_id,
|
| 80 |
+
workbook_path=wb_path,
|
| 81 |
+
)
|
| 82 |
+
self._engine.load_workbook(session)
|
| 83 |
+
self._scenario = scenario_def
|
| 84 |
+
self._last_validate_passed = 0
|
| 85 |
+
|
| 86 |
+
sheets = self._engine.list_sheets(self._session_id)
|
| 87 |
+
targets = self._engine.get_named_targets(self._session_id)
|
| 88 |
+
|
| 89 |
+
return {
|
| 90 |
+
"scenario_id": scenario_id,
|
| 91 |
+
"description": scenario_def.get("description", ""),
|
| 92 |
+
"instructions": scenario_def.get("instructions", ""),
|
| 93 |
+
"max_steps": scenario_def.get("max_steps", 50),
|
| 94 |
+
"sheets": sheets,
|
| 95 |
+
"target_regions": targets,
|
| 96 |
+
}
|
| 97 |
+
|
| 98 |
+
# ── Tool 4: list_sheets ───────────────────────────────────────
|
| 99 |
+
|
| 100 |
+
@mcp.tool()
|
| 101 |
+
def list_sheets() -> dict:
|
| 102 |
+
"""List all sheets in the current workbook with their names, row/column dimensions, and visibility state.
|
| 103 |
+
|
| 104 |
+
Returns an error if no scenario is loaded.
|
| 105 |
+
"""
|
| 106 |
+
_record("list_sheets")
|
| 107 |
+
if not self._session_id or self._session_id not in self._engine._sessions:
|
| 108 |
+
return {"error": "No workbook loaded. Use load_scenario first."}
|
| 109 |
+
sheets = self._engine.list_sheets(self._session_id)
|
| 110 |
+
return {"sheets": sheets}
|
| 111 |
+
|
| 112 |
+
# ── Tool 5: read_range ────────────────────────────────────────
|
| 113 |
+
|
| 114 |
+
@mcp.tool()
|
| 115 |
+
def read_range(sheet: str, range: str) -> dict:
|
| 116 |
+
"""Read a rectangular range of cells from a sheet.
|
| 117 |
+
|
| 118 |
+
Args:
|
| 119 |
+
sheet: Sheet name (e.g. "Summary", "Engineering").
|
| 120 |
+
range: Cell range in A1 notation (e.g. "A1", "B2:D10", "A1:Z100").
|
| 121 |
+
|
| 122 |
+
Returns a 2D array of cell values. Formulas are shown as their formula strings (e.g. "=SUM(A1:A10)").
|
| 123 |
+
"""
|
| 124 |
+
_record("read_range", sheet=sheet, range=range)
|
| 125 |
+
if not self._session_id or self._session_id not in self._engine._sessions:
|
| 126 |
+
return {"error": "No workbook loaded. Use load_scenario first."}
|
| 127 |
+
try:
|
| 128 |
+
data = self._engine.read_range(self._session_id, sheet, range)
|
| 129 |
+
return {"sheet": sheet, "range": range, "data": data}
|
| 130 |
+
except (ValueError, KeyError) as e:
|
| 131 |
+
return {"error": str(e)}
|
| 132 |
+
|
| 133 |
+
# ── Tool 6: write_cell ────────────────────────────────────────
|
| 134 |
+
|
| 135 |
+
@mcp.tool()
|
| 136 |
+
def write_cell(sheet: str, cell: str, value: str) -> dict:
|
| 137 |
+
"""Write a value or formula to a single cell.
|
| 138 |
+
|
| 139 |
+
Args:
|
| 140 |
+
sheet: Sheet name.
|
| 141 |
+
cell: Cell reference in A1 notation (e.g. "C15").
|
| 142 |
+
value: The value to write. Use "=" prefix for formulas (e.g. "=SUM(A1:A10)").
|
| 143 |
+
Numeric strings are auto-converted to numbers.
|
| 144 |
+
|
| 145 |
+
Returns confirmation of the write.
|
| 146 |
+
"""
|
| 147 |
+
_record("write_cell", sheet=sheet, cell=cell, value=value)
|
| 148 |
+
if not self._session_id or self._session_id not in self._engine._sessions:
|
| 149 |
+
return {"error": "No workbook loaded. Use load_scenario first."}
|
| 150 |
+
try:
|
| 151 |
+
parsed = _parse_value(value)
|
| 152 |
+
result = self._engine.write_cell(self._session_id, sheet, cell, parsed)
|
| 153 |
+
return result
|
| 154 |
+
except (ValueError, KeyError) as e:
|
| 155 |
+
return {"error": str(e)}
|
| 156 |
+
|
| 157 |
+
# ── Tool 7: write_range ───────────────────────────────────────
|
| 158 |
+
|
| 159 |
+
@mcp.tool()
|
| 160 |
+
def write_range(sheet: str, start_cell: str, data: str) -> dict:
|
| 161 |
+
"""Write a 2D block of values starting from a cell.
|
| 162 |
+
|
| 163 |
+
Args:
|
| 164 |
+
sheet: Sheet name.
|
| 165 |
+
start_cell: Top-left cell in A1 notation (e.g. "A1").
|
| 166 |
+
data: JSON string of a 2D array, e.g. '[[1, 2], [3, 4]]'.
|
| 167 |
+
Use "=" prefix for formulas within cells.
|
| 168 |
+
|
| 169 |
+
Returns the range written and cell count.
|
| 170 |
+
"""
|
| 171 |
+
_record("write_range", sheet=sheet, start_cell=start_cell, data=data)
|
| 172 |
+
if not self._session_id or self._session_id not in self._engine._sessions:
|
| 173 |
+
return {"error": "No workbook loaded. Use load_scenario first."}
|
| 174 |
+
try:
|
| 175 |
+
parsed_data = json.loads(data)
|
| 176 |
+
if not isinstance(parsed_data, list):
|
| 177 |
+
return {"error": "data must be a JSON 2D array, e.g. '[[1, 2], [3, 4]]'"}
|
| 178 |
+
converted = [[_parse_value(str(v)) for v in row] for row in parsed_data]
|
| 179 |
+
result = self._engine.write_range(self._session_id, sheet, start_cell, converted)
|
| 180 |
+
return result
|
| 181 |
+
except json.JSONDecodeError:
|
| 182 |
+
return {"error": "Invalid JSON in data parameter."}
|
| 183 |
+
except (ValueError, KeyError) as e:
|
| 184 |
+
return {"error": str(e)}
|
| 185 |
+
|
| 186 |
+
# ── Tool 8: inspect_formula ───────────────────────────────────
|
| 187 |
+
|
| 188 |
+
@mcp.tool()
|
| 189 |
+
def inspect_formula(sheet: str, cell: str) -> dict:
|
| 190 |
+
"""Return the raw formula string from a cell, or indicate it's not a formula.
|
| 191 |
+
|
| 192 |
+
Args:
|
| 193 |
+
sheet: Sheet name.
|
| 194 |
+
cell: Cell reference (e.g. "C15").
|
| 195 |
+
|
| 196 |
+
Returns the formula string if the cell contains one, or is_formula=false otherwise.
|
| 197 |
+
"""
|
| 198 |
+
_record("inspect_formula", sheet=sheet, cell=cell)
|
| 199 |
+
if not self._session_id or self._session_id not in self._engine._sessions:
|
| 200 |
+
return {"error": "No workbook loaded. Use load_scenario first."}
|
| 201 |
+
try:
|
| 202 |
+
return self._engine.inspect_formula(self._session_id, sheet, cell)
|
| 203 |
+
except (ValueError, KeyError) as e:
|
| 204 |
+
return {"error": str(e)}
|
| 205 |
+
|
| 206 |
+
# ── Tool 9: list_named_targets ────────────────────────────────
|
| 207 |
+
|
| 208 |
+
@mcp.tool()
|
| 209 |
+
def list_named_targets() -> dict:
|
| 210 |
+
"""Show the target areas and allowed output zones for the current scenario.
|
| 211 |
+
|
| 212 |
+
Target regions are the cells/ranges where the agent is expected to write.
|
| 213 |
+
Writing outside these areas may incur a penalty.
|
| 214 |
+
"""
|
| 215 |
+
_record("list_named_targets")
|
| 216 |
+
if not self._session_id or self._session_id not in self._engine._sessions:
|
| 217 |
+
return {"error": "No workbook loaded. Use load_scenario first."}
|
| 218 |
+
targets = self._engine.get_named_targets(self._session_id)
|
| 219 |
+
return {"target_regions": targets}
|
| 220 |
+
|
| 221 |
+
# ── Tool 10: validate_partial ─────────────────────────────────
|
| 222 |
+
|
| 223 |
+
@mcp.tool()
|
| 224 |
+
def validate_partial() -> dict:
|
| 225 |
+
"""Check partial progress on the current scenario.
|
| 226 |
+
|
| 227 |
+
Returns the number of hidden test checks that pass and fail,
|
| 228 |
+
without revealing the specific expected answers. Use this to
|
| 229 |
+
gauge progress before submitting.
|
| 230 |
+
"""
|
| 231 |
+
_record("validate_partial")
|
| 232 |
+
if not self._session_id or self._session_id not in self._engine._sessions:
|
| 233 |
+
return {"error": "No workbook loaded. Use load_scenario first."}
|
| 234 |
+
result = self._engine.validate_partial(self._session_id)
|
| 235 |
+
self._last_validate_passed = result.get("passed", 0)
|
| 236 |
+
return result
|
| 237 |
+
|
| 238 |
+
# ── Tool 11: submit_workbook ──────────────────────────────────
|
| 239 |
+
|
| 240 |
+
@mcp.tool()
|
| 241 |
+
def submit_workbook() -> dict:
|
| 242 |
+
"""Submit the workbook for final evaluation against hidden tests.
|
| 243 |
+
|
| 244 |
+
Runs all hidden test checks and returns structured results including
|
| 245 |
+
pass rate, per-check pass/fail, and whether the scenario is fully solved.
|
| 246 |
+
"""
|
| 247 |
+
_record("submit_workbook")
|
| 248 |
+
if not self._session_id or self._session_id not in self._engine._sessions:
|
| 249 |
+
return {"error": "No workbook loaded. Use load_scenario first."}
|
| 250 |
+
result = self._engine.run_hidden_tests(self._session_id)
|
| 251 |
+
return result
|
| 252 |
+
|
| 253 |
+
# ── Tool 12: get_edit_history ─────────────────────────────────
|
| 254 |
+
|
| 255 |
+
@mcp.tool()
|
| 256 |
+
def get_edit_history() -> dict:
|
| 257 |
+
"""Return the full list of cell edits made in this session, in order.
|
| 258 |
+
|
| 259 |
+
Each entry shows the sheet, cell, value written, and the step number.
|
| 260 |
+
"""
|
| 261 |
+
_record("get_edit_history")
|
| 262 |
+
if not self._session_id or self._session_id not in self._engine._sessions:
|
| 263 |
+
return {"error": "No workbook loaded. Use load_scenario first."}
|
| 264 |
+
history = self._engine.get_edit_history(self._session_id)
|
| 265 |
+
return {"edits": history, "count": len(history)}
|
| 266 |
+
|
| 267 |
+
# ── Tool 13: reset_scenario ───────────────────────────────────
|
| 268 |
+
|
| 269 |
+
@mcp.tool()
|
| 270 |
+
def reset_scenario() -> dict:
|
| 271 |
+
"""Restore the workbook to its original state, discarding all edits.
|
| 272 |
+
|
| 273 |
+
The scenario remains loaded; you do not need to call load_scenario again.
|
| 274 |
+
"""
|
| 275 |
+
_record("reset_scenario")
|
| 276 |
+
if not self._session_id or self._session_id not in self._engine._sessions:
|
| 277 |
+
return {"error": "No workbook loaded. Use load_scenario first."}
|
| 278 |
+
self._engine.reset_workbook(self._session_id)
|
| 279 |
+
self._last_validate_passed = 0
|
| 280 |
+
sheets = self._engine.list_sheets(self._session_id)
|
| 281 |
+
return {"message": "Workbook reset to original state.", "sheets": sheets}
|
| 282 |
+
|
| 283 |
+
super().__init__(mcp)
|
| 284 |
+
|
| 285 |
+
# ── Lifecycle ─────────────────────────────────────────────────────
|
| 286 |
+
|
| 287 |
+
def reset(
|
| 288 |
+
self,
|
| 289 |
+
seed: Optional[int] = None,
|
| 290 |
+
episode_id: Optional[str] = None,
|
| 291 |
+
**kwargs: Any,
|
| 292 |
+
) -> Observation:
|
| 293 |
+
if self._session_id and self._session_id in self._engine._sessions:
|
| 294 |
+
self._engine.close_session(self._session_id)
|
| 295 |
+
|
| 296 |
+
self._session_id = str(uuid4())
|
| 297 |
+
self._state = State(
|
| 298 |
+
episode_id=episode_id or self._session_id,
|
| 299 |
+
step_count=0,
|
| 300 |
+
)
|
| 301 |
+
self._scenario = None
|
| 302 |
+
self._action_history = []
|
| 303 |
+
self._last_validate_passed = 0
|
| 304 |
+
|
| 305 |
+
return Observation(
|
| 306 |
+
done=False,
|
| 307 |
+
reward=0.0,
|
| 308 |
+
metadata={
|
| 309 |
+
"status": "ready",
|
| 310 |
+
"session_id": self._session_id,
|
| 311 |
+
"instructions": (
|
| 312 |
+
"Use list_scenarios to see available tasks, then load_scenario to begin. "
|
| 313 |
+
"Read the workbook structure with list_sheets and read_range before making edits. "
|
| 314 |
+
"Use submit_workbook when done."
|
| 315 |
+
),
|
| 316 |
+
},
|
| 317 |
+
)
|
| 318 |
+
|
| 319 |
+
def step(self, action: Action, timeout_s: Optional[float] = None, **kwargs: Any) -> Observation:
|
| 320 |
+
self._state.step_count += 1
|
| 321 |
+
if hasattr(action, "to_mcp_action"):
|
| 322 |
+
action = action.to_mcp_action()
|
| 323 |
+
|
| 324 |
+
obs = super().step(action, timeout_s=timeout_s, **kwargs)
|
| 325 |
+
|
| 326 |
+
tool_name = getattr(action, "tool_name", None)
|
| 327 |
+
args = getattr(action, "arguments", None) or {}
|
| 328 |
+
result = getattr(obs, "result", None)
|
| 329 |
+
if hasattr(result, "data"):
|
| 330 |
+
result = result.data
|
| 331 |
+
elif isinstance(result, dict) and "data" in result:
|
| 332 |
+
result = result["data"]
|
| 333 |
+
if not isinstance(result, dict):
|
| 334 |
+
result = {}
|
| 335 |
+
|
| 336 |
+
reward = self._compute_step_reward(tool_name, args, result)
|
| 337 |
+
if reward != 0:
|
| 338 |
+
obs.reward = (obs.reward or 0) + reward
|
| 339 |
+
|
| 340 |
+
session = self._engine._sessions.get(self._session_id)
|
| 341 |
+
if session:
|
| 342 |
+
obs.done = session.solved
|
| 343 |
+
|
| 344 |
+
return obs
|
| 345 |
+
|
| 346 |
+
def _compute_step_reward(self, tool_name: Optional[str], args: dict, result: dict) -> float:
|
| 347 |
+
"""Layer 1 per-step reward heuristics (internal, approximate)."""
|
| 348 |
+
if isinstance(result, dict) and result.get("error"):
|
| 349 |
+
return 0.0
|
| 350 |
+
|
| 351 |
+
if tool_name == "inspect_formula":
|
| 352 |
+
return 0.05
|
| 353 |
+
|
| 354 |
+
if tool_name == "validate_partial":
|
| 355 |
+
new_passed = result.get("passed", 0)
|
| 356 |
+
if new_passed > self._last_validate_passed:
|
| 357 |
+
return 0.10
|
| 358 |
+
return 0.05
|
| 359 |
+
|
| 360 |
+
if tool_name in WRITE_TOOLS:
|
| 361 |
+
sheet = args.get("sheet", "")
|
| 362 |
+
cell = args.get("cell", args.get("start_cell", ""))
|
| 363 |
+
|
| 364 |
+
in_target = True
|
| 365 |
+
if self._session_id and self._session_id in self._engine._sessions:
|
| 366 |
+
in_target = self._engine.is_in_target_region(self._session_id, sheet, cell)
|
| 367 |
+
|
| 368 |
+
if not in_target:
|
| 369 |
+
return -0.10
|
| 370 |
+
|
| 371 |
+
recent_reads = any(
|
| 372 |
+
a["tool"] in ("read_range", "read_cell")
|
| 373 |
+
for a in self._action_history[-4:-1]
|
| 374 |
+
)
|
| 375 |
+
reward = 0.05
|
| 376 |
+
if recent_reads:
|
| 377 |
+
reward += 0.05
|
| 378 |
+
|
| 379 |
+
if self._session_id and self._session_id in self._engine._sessions:
|
| 380 |
+
cell_ref = cell.upper()
|
| 381 |
+
write_count = sum(
|
| 382 |
+
1 for a in self._action_history
|
| 383 |
+
if a["tool"] in WRITE_TOOLS
|
| 384 |
+
and a["arguments"].get("cell", a["arguments"].get("start_cell", "")).upper() == cell_ref
|
| 385 |
+
and a["arguments"].get("sheet", "") == sheet
|
| 386 |
+
)
|
| 387 |
+
if write_count >= 3:
|
| 388 |
+
reward -= 0.05
|
| 389 |
+
|
| 390 |
+
return reward
|
| 391 |
+
|
| 392 |
+
if tool_name in READ_TOOLS:
|
| 393 |
+
return 0.0
|
| 394 |
+
|
| 395 |
+
if tool_name == "submit_workbook":
|
| 396 |
+
pass_rate = result.get("pass_rate", 0)
|
| 397 |
+
if pass_rate == 1.0:
|
| 398 |
+
return 0.50
|
| 399 |
+
if pass_rate > 0.5:
|
| 400 |
+
return 0.20
|
| 401 |
+
if pass_rate < 0.3:
|
| 402 |
+
return -0.10
|
| 403 |
+
return 0.0
|
| 404 |
+
|
| 405 |
+
return 0.0
|
| 406 |
+
|
| 407 |
+
def _step_impl(self, action: Action, timeout_s: Optional[float] = None, **kwargs: Any) -> Observation:
|
| 408 |
+
return Observation(
|
| 409 |
+
done=False,
|
| 410 |
+
reward=0.0,
|
| 411 |
+
metadata={
|
| 412 |
+
"error": f"Unknown action type: {type(action).__name__}. "
|
| 413 |
+
"Use ListToolsAction or CallToolAction."
|
| 414 |
+
},
|
| 415 |
+
)
|
| 416 |
+
|
| 417 |
+
@property
|
| 418 |
+
def state(self) -> State:
|
| 419 |
+
return self._state
|
| 420 |
+
|
| 421 |
+
def get_metadata(self) -> EnvironmentMetadata:
|
| 422 |
+
return EnvironmentMetadata(
|
| 423 |
+
name="spreadsheet",
|
| 424 |
+
description="Spreadsheet — exact workbook manipulation and reasoning over realistic spreadsheet tasks",
|
| 425 |
+
version="0.1.0",
|
| 426 |
+
)
|
| 427 |
+
|
| 428 |
+
|
| 429 |
+
def _parse_value(value: str) -> Any:
|
| 430 |
+
"""Convert string input to appropriate Python type for cell writing."""
|
| 431 |
+
if isinstance(value, str) and value.startswith("="):
|
| 432 |
+
return value
|
| 433 |
+
try:
|
| 434 |
+
if "." in value:
|
| 435 |
+
return float(value)
|
| 436 |
+
return int(value)
|
| 437 |
+
except (ValueError, TypeError):
|
| 438 |
+
pass
|
| 439 |
+
if value.lower() in ("true",):
|
| 440 |
+
return True
|
| 441 |
+
if value.lower() in ("false",):
|
| 442 |
+
return False
|
| 443 |
+
if value.lower() in ("none", "null", ""):
|
| 444 |
+
return None
|
| 445 |
+
return value
|
server/workbook_engine.py
ADDED
|
@@ -0,0 +1,564 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Workbook engine — load, edit, and validate Excel workbooks via openpyxl."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
import copy
|
| 6 |
+
import json
|
| 7 |
+
import os
|
| 8 |
+
import re
|
| 9 |
+
import shutil
|
| 10 |
+
from datetime import date, datetime
|
| 11 |
+
from pathlib import Path
|
| 12 |
+
from typing import Any, Optional
|
| 13 |
+
|
| 14 |
+
import openpyxl
|
| 15 |
+
from openpyxl.utils import get_column_letter, column_index_from_string
|
| 16 |
+
from pydantic import BaseModel
|
| 17 |
+
|
| 18 |
+
WORKBOOKS_DIR = os.getenv("WORKBOOKS_DIR", str(Path(__file__).resolve().parent.parent / "workbooks"))
|
| 19 |
+
HIDDEN_TESTS_DIR = os.path.join(WORKBOOKS_DIR, "hidden_tests")
|
| 20 |
+
FIXTURES_DIR = os.path.join(WORKBOOKS_DIR, "fixtures")
|
| 21 |
+
TEMPLATES_DIR = os.path.join(WORKBOOKS_DIR, "templates")
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
class WorkbookSession(BaseModel):
|
| 25 |
+
session_id: str
|
| 26 |
+
scenario_id: str
|
| 27 |
+
workbook_path: str
|
| 28 |
+
modified_cells: list[dict] = []
|
| 29 |
+
step_count: int = 0
|
| 30 |
+
solved: bool = False
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
def _parse_cell_ref(ref: str) -> tuple[str, int]:
|
| 34 |
+
"""Parse 'A1' into (column_letter, row_number)."""
|
| 35 |
+
m = re.match(r"^([A-Z]+)(\d+)$", ref.upper().strip())
|
| 36 |
+
if not m:
|
| 37 |
+
raise ValueError(f"Invalid cell reference: {ref}")
|
| 38 |
+
return m.group(1), int(m.group(2))
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
def _parse_range_ref(range_str: str) -> tuple[str, str]:
|
| 42 |
+
"""Parse 'A1:D10' into ('A1', 'D10')."""
|
| 43 |
+
parts = range_str.upper().strip().split(":")
|
| 44 |
+
if len(parts) == 1:
|
| 45 |
+
return parts[0], parts[0]
|
| 46 |
+
if len(parts) == 2:
|
| 47 |
+
return parts[0], parts[1]
|
| 48 |
+
raise ValueError(f"Invalid range reference: {range_str}")
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
class WorkbookEngine:
|
| 52 |
+
"""In-memory workbook operations backed by openpyxl."""
|
| 53 |
+
|
| 54 |
+
def __init__(self):
|
| 55 |
+
self._sessions: dict[str, WorkbookSession] = {}
|
| 56 |
+
self._workbooks: dict[str, openpyxl.Workbook] = {}
|
| 57 |
+
|
| 58 |
+
def load_workbook(self, session: WorkbookSession) -> None:
|
| 59 |
+
"""Load a workbook from disk into memory for a session."""
|
| 60 |
+
if not os.path.isfile(session.workbook_path):
|
| 61 |
+
raise FileNotFoundError(f"Workbook not found: {session.workbook_path}")
|
| 62 |
+
wb = openpyxl.load_workbook(session.workbook_path, data_only=False)
|
| 63 |
+
self._sessions[session.session_id] = session
|
| 64 |
+
self._workbooks[session.session_id] = wb
|
| 65 |
+
|
| 66 |
+
def reset_workbook(self, session_id: str) -> None:
|
| 67 |
+
"""Reload the original workbook from disk, discarding all edits."""
|
| 68 |
+
session = self._get_session(session_id)
|
| 69 |
+
session.modified_cells = []
|
| 70 |
+
session.step_count = 0
|
| 71 |
+
session.solved = False
|
| 72 |
+
wb = openpyxl.load_workbook(session.workbook_path, data_only=False)
|
| 73 |
+
self._workbooks[session_id] = wb
|
| 74 |
+
|
| 75 |
+
def close_session(self, session_id: str) -> None:
|
| 76 |
+
"""Remove a session and free its workbook."""
|
| 77 |
+
self._sessions.pop(session_id, None)
|
| 78 |
+
self._workbooks.pop(session_id, None)
|
| 79 |
+
|
| 80 |
+
def list_sheets(self, session_id: str) -> list[dict]:
|
| 81 |
+
"""Return sheet names with basic metadata."""
|
| 82 |
+
wb = self._get_wb(session_id)
|
| 83 |
+
sheets = []
|
| 84 |
+
for name in wb.sheetnames:
|
| 85 |
+
ws = wb[name]
|
| 86 |
+
sheets.append({
|
| 87 |
+
"name": name,
|
| 88 |
+
"min_row": ws.min_row,
|
| 89 |
+
"max_row": ws.max_row,
|
| 90 |
+
"min_column": ws.min_column,
|
| 91 |
+
"max_column": ws.max_column,
|
| 92 |
+
"state": ws.sheet_state,
|
| 93 |
+
})
|
| 94 |
+
return sheets
|
| 95 |
+
|
| 96 |
+
def read_range(self, session_id: str, sheet: str, range_str: str) -> list[list[Any]]:
|
| 97 |
+
"""Read a rectangular range and return a 2D list of cell values."""
|
| 98 |
+
ws = self._get_sheet(session_id, sheet)
|
| 99 |
+
start, end = _parse_range_ref(range_str)
|
| 100 |
+
start_col, start_row = _parse_cell_ref(start)
|
| 101 |
+
end_col, end_row = _parse_cell_ref(end)
|
| 102 |
+
|
| 103 |
+
min_col = column_index_from_string(start_col)
|
| 104 |
+
max_col = column_index_from_string(end_col)
|
| 105 |
+
|
| 106 |
+
rows = []
|
| 107 |
+
for r in range(start_row, end_row + 1):
|
| 108 |
+
row_data = []
|
| 109 |
+
for c in range(min_col, max_col + 1):
|
| 110 |
+
cell = ws.cell(row=r, column=c)
|
| 111 |
+
row_data.append(self._cell_display_value(cell))
|
| 112 |
+
rows.append(row_data)
|
| 113 |
+
return rows
|
| 114 |
+
|
| 115 |
+
def read_cell(self, session_id: str, sheet: str, cell_ref: str) -> dict:
|
| 116 |
+
"""Read a single cell and return value, formula, and type info."""
|
| 117 |
+
ws = self._get_sheet(session_id, sheet)
|
| 118 |
+
col_letter, row_num = _parse_cell_ref(cell_ref)
|
| 119 |
+
col_idx = column_index_from_string(col_letter)
|
| 120 |
+
cell = ws.cell(row=row_num, column=col_idx)
|
| 121 |
+
|
| 122 |
+
return {
|
| 123 |
+
"cell": cell_ref.upper(),
|
| 124 |
+
"value": self._cell_display_value(cell),
|
| 125 |
+
"formula": cell.value if isinstance(cell.value, str) and cell.value.startswith("=") else None,
|
| 126 |
+
"data_type": cell.data_type,
|
| 127 |
+
"number_format": cell.number_format,
|
| 128 |
+
}
|
| 129 |
+
|
| 130 |
+
def inspect_formula(self, session_id: str, sheet: str, cell_ref: str) -> dict:
|
| 131 |
+
"""Return the raw formula string from a cell, or None if not a formula."""
|
| 132 |
+
ws = self._get_sheet(session_id, sheet)
|
| 133 |
+
col_letter, row_num = _parse_cell_ref(cell_ref)
|
| 134 |
+
col_idx = column_index_from_string(col_letter)
|
| 135 |
+
cell = ws.cell(row=row_num, column=col_idx)
|
| 136 |
+
|
| 137 |
+
raw = cell.value
|
| 138 |
+
is_formula = isinstance(raw, str) and raw.startswith("=")
|
| 139 |
+
return {
|
| 140 |
+
"cell": cell_ref.upper(),
|
| 141 |
+
"formula": raw if is_formula else None,
|
| 142 |
+
"is_formula": is_formula,
|
| 143 |
+
}
|
| 144 |
+
|
| 145 |
+
def write_cell(self, session_id: str, sheet: str, cell_ref: str, value: Any) -> dict:
|
| 146 |
+
"""Write a value or formula to a single cell."""
|
| 147 |
+
session = self._get_session(session_id)
|
| 148 |
+
ws = self._get_sheet(session_id, sheet)
|
| 149 |
+
col_letter, row_num = _parse_cell_ref(cell_ref)
|
| 150 |
+
col_idx = column_index_from_string(col_letter)
|
| 151 |
+
|
| 152 |
+
ws.cell(row=row_num, column=col_idx, value=value)
|
| 153 |
+
session.modified_cells.append({
|
| 154 |
+
"sheet": sheet,
|
| 155 |
+
"cell": cell_ref.upper(),
|
| 156 |
+
"value": str(value),
|
| 157 |
+
"step": session.step_count,
|
| 158 |
+
})
|
| 159 |
+
return {"written": cell_ref.upper(), "sheet": sheet, "value": str(value)}
|
| 160 |
+
|
| 161 |
+
def write_range(self, session_id: str, sheet: str, start_cell: str, data: list[list[Any]]) -> dict:
|
| 162 |
+
"""Write a 2D block of values starting from start_cell."""
|
| 163 |
+
session = self._get_session(session_id)
|
| 164 |
+
ws = self._get_sheet(session_id, sheet)
|
| 165 |
+
col_letter, start_row = _parse_cell_ref(start_cell)
|
| 166 |
+
start_col = column_index_from_string(col_letter)
|
| 167 |
+
|
| 168 |
+
cells_written = 0
|
| 169 |
+
for r_offset, row_data in enumerate(data):
|
| 170 |
+
for c_offset, val in enumerate(row_data):
|
| 171 |
+
row_num = start_row + r_offset
|
| 172 |
+
col_idx = start_col + c_offset
|
| 173 |
+
ws.cell(row=row_num, column=col_idx, value=val)
|
| 174 |
+
cell_ref = f"{get_column_letter(col_idx)}{row_num}"
|
| 175 |
+
session.modified_cells.append({
|
| 176 |
+
"sheet": sheet,
|
| 177 |
+
"cell": cell_ref,
|
| 178 |
+
"value": str(val),
|
| 179 |
+
"step": session.step_count,
|
| 180 |
+
})
|
| 181 |
+
cells_written += 1
|
| 182 |
+
|
| 183 |
+
end_row = start_row + len(data) - 1
|
| 184 |
+
end_col = start_col + (max(len(r) for r in data) - 1 if data else 0)
|
| 185 |
+
end_ref = f"{get_column_letter(end_col)}{end_row}"
|
| 186 |
+
return {
|
| 187 |
+
"range": f"{start_cell.upper()}:{end_ref}",
|
| 188 |
+
"sheet": sheet,
|
| 189 |
+
"cells_written": cells_written,
|
| 190 |
+
}
|
| 191 |
+
|
| 192 |
+
def copy_range(
|
| 193 |
+
self, session_id: str,
|
| 194 |
+
src_sheet: str, src_range: str,
|
| 195 |
+
dst_sheet: str, dst_start: str,
|
| 196 |
+
) -> dict:
|
| 197 |
+
"""Copy a range of cells from one location to another (values and formulas)."""
|
| 198 |
+
data = self.read_range(session_id, src_sheet, src_range)
|
| 199 |
+
src_ws = self._get_sheet(session_id, src_sheet)
|
| 200 |
+
start_ref, end_ref = _parse_range_ref(src_range)
|
| 201 |
+
start_col_letter, start_row = _parse_cell_ref(start_ref)
|
| 202 |
+
end_col_letter, end_row = _parse_cell_ref(end_ref)
|
| 203 |
+
min_col = column_index_from_string(start_col_letter)
|
| 204 |
+
max_col = column_index_from_string(end_col_letter)
|
| 205 |
+
|
| 206 |
+
raw_data = []
|
| 207 |
+
for r in range(start_row, end_row + 1):
|
| 208 |
+
row = []
|
| 209 |
+
for c in range(min_col, max_col + 1):
|
| 210 |
+
cell = src_ws.cell(row=r, column=c)
|
| 211 |
+
row.append(cell.value)
|
| 212 |
+
raw_data.append(row)
|
| 213 |
+
|
| 214 |
+
result = self.write_range(session_id, dst_sheet, dst_start, raw_data)
|
| 215 |
+
return {"copied_from": f"{src_sheet}!{src_range}", **result}
|
| 216 |
+
|
| 217 |
+
def get_edit_history(self, session_id: str) -> list[dict]:
|
| 218 |
+
"""Return the list of all edits made in this session."""
|
| 219 |
+
session = self._get_session(session_id)
|
| 220 |
+
return list(session.modified_cells)
|
| 221 |
+
|
| 222 |
+
def get_session_info(self, session_id: str) -> dict:
|
| 223 |
+
"""Return session metadata."""
|
| 224 |
+
session = self._get_session(session_id)
|
| 225 |
+
return {
|
| 226 |
+
"session_id": session.session_id,
|
| 227 |
+
"scenario_id": session.scenario_id,
|
| 228 |
+
"step_count": session.step_count,
|
| 229 |
+
"edits_made": len(session.modified_cells),
|
| 230 |
+
"solved": session.solved,
|
| 231 |
+
}
|
| 232 |
+
|
| 233 |
+
# ── Hidden test execution ──────────────────────────────────────────
|
| 234 |
+
|
| 235 |
+
def run_hidden_tests(self, session_id: str) -> dict:
|
| 236 |
+
"""Run all hidden test checks for the current scenario and return results."""
|
| 237 |
+
session = self._get_session(session_id)
|
| 238 |
+
wb = self._get_wb(session_id)
|
| 239 |
+
|
| 240 |
+
test_path = os.path.join(HIDDEN_TESTS_DIR, f"{session.scenario_id}.json")
|
| 241 |
+
if not os.path.isfile(test_path):
|
| 242 |
+
return {"error": f"No hidden tests found for scenario {session.scenario_id}"}
|
| 243 |
+
|
| 244 |
+
with open(test_path) as f:
|
| 245 |
+
test_spec = json.load(f)
|
| 246 |
+
|
| 247 |
+
checks = test_spec.get("checks", [])
|
| 248 |
+
results = []
|
| 249 |
+
passed = 0
|
| 250 |
+
|
| 251 |
+
for check in checks:
|
| 252 |
+
result = self._run_single_check(wb, check)
|
| 253 |
+
results.append(result)
|
| 254 |
+
if result["passed"]:
|
| 255 |
+
passed += 1
|
| 256 |
+
|
| 257 |
+
total = len(checks)
|
| 258 |
+
pass_rate = passed / total if total > 0 else 0.0
|
| 259 |
+
session.solved = pass_rate == 1.0
|
| 260 |
+
|
| 261 |
+
return {
|
| 262 |
+
"scenario_id": session.scenario_id,
|
| 263 |
+
"total_checks": total,
|
| 264 |
+
"passed": passed,
|
| 265 |
+
"failed": total - passed,
|
| 266 |
+
"pass_rate": pass_rate,
|
| 267 |
+
"results": results,
|
| 268 |
+
}
|
| 269 |
+
|
| 270 |
+
def validate_partial(self, session_id: str) -> dict:
|
| 271 |
+
"""Run hidden tests but return only pass/fail counts, not full answers."""
|
| 272 |
+
full = self.run_hidden_tests(session_id)
|
| 273 |
+
if "error" in full:
|
| 274 |
+
return full
|
| 275 |
+
return {
|
| 276 |
+
"scenario_id": full["scenario_id"],
|
| 277 |
+
"total_checks": full["total_checks"],
|
| 278 |
+
"passed": full["passed"],
|
| 279 |
+
"failed": full["failed"],
|
| 280 |
+
"pass_rate": full["pass_rate"],
|
| 281 |
+
}
|
| 282 |
+
|
| 283 |
+
# ── Target region helpers ──────────────────────────────────────────
|
| 284 |
+
|
| 285 |
+
def get_named_targets(self, session_id: str) -> list[dict]:
|
| 286 |
+
"""Return scenario-defined target areas where the agent should write."""
|
| 287 |
+
session = self._get_session(session_id)
|
| 288 |
+
test_path = os.path.join(HIDDEN_TESTS_DIR, f"{session.scenario_id}.json")
|
| 289 |
+
if not os.path.isfile(test_path):
|
| 290 |
+
return []
|
| 291 |
+
with open(test_path) as f:
|
| 292 |
+
test_spec = json.load(f)
|
| 293 |
+
return test_spec.get("target_regions", [])
|
| 294 |
+
|
| 295 |
+
def is_in_target_region(self, session_id: str, sheet: str, cell_ref: str) -> bool:
|
| 296 |
+
"""Check if a cell is within a designated target region."""
|
| 297 |
+
targets = self.get_named_targets(session_id)
|
| 298 |
+
if not targets:
|
| 299 |
+
return True
|
| 300 |
+
cell_ref = cell_ref.upper()
|
| 301 |
+
for t in targets:
|
| 302 |
+
if t.get("sheet") != sheet:
|
| 303 |
+
continue
|
| 304 |
+
t_range = t.get("range")
|
| 305 |
+
if t_range and self._cell_in_range(cell_ref, t_range):
|
| 306 |
+
return True
|
| 307 |
+
return False
|
| 308 |
+
|
| 309 |
+
# ── Private helpers ────────────────────────────────────────────────
|
| 310 |
+
|
| 311 |
+
def _get_session(self, session_id: str) -> WorkbookSession:
|
| 312 |
+
if session_id not in self._sessions:
|
| 313 |
+
raise KeyError(f"Session not found: {session_id}")
|
| 314 |
+
return self._sessions[session_id]
|
| 315 |
+
|
| 316 |
+
def _get_wb(self, session_id: str) -> openpyxl.Workbook:
|
| 317 |
+
if session_id not in self._workbooks:
|
| 318 |
+
raise KeyError(f"No workbook loaded for session: {session_id}")
|
| 319 |
+
return self._workbooks[session_id]
|
| 320 |
+
|
| 321 |
+
def _get_sheet(self, session_id: str, sheet_name: str):
|
| 322 |
+
wb = self._get_wb(session_id)
|
| 323 |
+
if sheet_name not in wb.sheetnames:
|
| 324 |
+
raise ValueError(f"Sheet '{sheet_name}' not found. Available: {wb.sheetnames}")
|
| 325 |
+
return wb[sheet_name]
|
| 326 |
+
|
| 327 |
+
def _cell_display_value(self, cell) -> Any:
|
| 328 |
+
"""Return a JSON-safe display value for a cell."""
|
| 329 |
+
val = cell.value
|
| 330 |
+
if val is None:
|
| 331 |
+
return None
|
| 332 |
+
if isinstance(val, str) and val.startswith("="):
|
| 333 |
+
return val
|
| 334 |
+
if isinstance(val, (datetime, date)):
|
| 335 |
+
return val.isoformat()
|
| 336 |
+
return val
|
| 337 |
+
|
| 338 |
+
def _cell_in_range(self, cell_ref: str, range_str: str) -> bool:
|
| 339 |
+
"""Check if cell_ref falls within range_str (e.g. 'B2:D10')."""
|
| 340 |
+
start, end = _parse_range_ref(range_str)
|
| 341 |
+
s_col, s_row = _parse_cell_ref(start)
|
| 342 |
+
e_col, e_row = _parse_cell_ref(end)
|
| 343 |
+
c_col, c_row = _parse_cell_ref(cell_ref)
|
| 344 |
+
|
| 345 |
+
return (
|
| 346 |
+
column_index_from_string(s_col) <= column_index_from_string(c_col) <= column_index_from_string(e_col)
|
| 347 |
+
and s_row <= c_row <= e_row
|
| 348 |
+
)
|
| 349 |
+
|
| 350 |
+
def _run_single_check(self, wb: openpyxl.Workbook, check: dict) -> dict:
|
| 351 |
+
"""Execute a single hidden test check against the workbook."""
|
| 352 |
+
check_type = self._determine_check_type(check)
|
| 353 |
+
try:
|
| 354 |
+
if check_type == "expected_formula":
|
| 355 |
+
return self._check_expected_formula(wb, check)
|
| 356 |
+
elif check_type == "expected_value_range":
|
| 357 |
+
return self._check_expected_value_range(wb, check)
|
| 358 |
+
elif check_type == "no_blanks":
|
| 359 |
+
return self._check_no_blanks(wb, check)
|
| 360 |
+
elif check_type == "row_count_equals":
|
| 361 |
+
return self._check_row_count_equals(wb, check)
|
| 362 |
+
elif check_type == "all_dates_iso_format":
|
| 363 |
+
return self._check_all_dates_iso(wb, check)
|
| 364 |
+
elif check_type == "constraint_satisfaction":
|
| 365 |
+
return self._check_constraint_satisfaction(wb, check)
|
| 366 |
+
else:
|
| 367 |
+
return {"check": check_type, "passed": False, "reason": f"Unknown check type: {check_type}"}
|
| 368 |
+
except Exception as e:
|
| 369 |
+
return {"check": check_type, "passed": False, "reason": str(e)}
|
| 370 |
+
|
| 371 |
+
def _determine_check_type(self, check: dict) -> str:
|
| 372 |
+
if "expected_formula" in check:
|
| 373 |
+
return "expected_formula"
|
| 374 |
+
if "expected_value_range" in check:
|
| 375 |
+
return "expected_value_range"
|
| 376 |
+
return check.get("check", "unknown")
|
| 377 |
+
|
| 378 |
+
def _check_expected_formula(self, wb: openpyxl.Workbook, check: dict) -> dict:
|
| 379 |
+
ws = wb[check["sheet"]]
|
| 380 |
+
col_letter, row_num = _parse_cell_ref(check["cell"])
|
| 381 |
+
col_idx = column_index_from_string(col_letter)
|
| 382 |
+
cell = ws.cell(row=row_num, column=col_idx)
|
| 383 |
+
actual = cell.value
|
| 384 |
+
expected = check["expected_formula"]
|
| 385 |
+
passed = isinstance(actual, str) and actual.strip() == expected.strip()
|
| 386 |
+
return {
|
| 387 |
+
"check": "expected_formula",
|
| 388 |
+
"cell": f"{check['sheet']}!{check['cell']}",
|
| 389 |
+
"passed": passed,
|
| 390 |
+
"expected": expected,
|
| 391 |
+
"actual": actual,
|
| 392 |
+
}
|
| 393 |
+
|
| 394 |
+
def _check_expected_value_range(self, wb: openpyxl.Workbook, check: dict) -> dict:
|
| 395 |
+
ws = wb[check["sheet"]]
|
| 396 |
+
col_letter, row_num = _parse_cell_ref(check["cell"])
|
| 397 |
+
col_idx = column_index_from_string(col_letter)
|
| 398 |
+
cell = ws.cell(row=row_num, column=col_idx)
|
| 399 |
+
val = cell.value
|
| 400 |
+
|
| 401 |
+
lo, hi = check["expected_value_range"]
|
| 402 |
+
if isinstance(val, str) and val.startswith("="):
|
| 403 |
+
from .formula_utils import evaluate_formula
|
| 404 |
+
val = evaluate_formula(wb, check["sheet"], check["cell"])
|
| 405 |
+
|
| 406 |
+
numeric = self._to_numeric(val)
|
| 407 |
+
if numeric is None:
|
| 408 |
+
return {
|
| 409 |
+
"check": "expected_value_range",
|
| 410 |
+
"cell": f"{check['sheet']}!{check['cell']}",
|
| 411 |
+
"passed": False,
|
| 412 |
+
"reason": f"Non-numeric value: {val}",
|
| 413 |
+
}
|
| 414 |
+
passed = lo <= numeric <= hi
|
| 415 |
+
return {
|
| 416 |
+
"check": "expected_value_range",
|
| 417 |
+
"cell": f"{check['sheet']}!{check['cell']}",
|
| 418 |
+
"passed": passed,
|
| 419 |
+
"expected_range": [lo, hi],
|
| 420 |
+
"actual": numeric,
|
| 421 |
+
}
|
| 422 |
+
|
| 423 |
+
def _check_no_blanks(self, wb: openpyxl.Workbook, check: dict) -> dict:
|
| 424 |
+
ws = wb[check["sheet"]]
|
| 425 |
+
range_str = check["range"]
|
| 426 |
+
start, end = _parse_range_ref(range_str)
|
| 427 |
+
s_col, s_row = _parse_cell_ref(start)
|
| 428 |
+
e_col, e_row = _parse_cell_ref(end)
|
| 429 |
+
min_col = column_index_from_string(s_col)
|
| 430 |
+
max_col = column_index_from_string(e_col)
|
| 431 |
+
|
| 432 |
+
blanks = []
|
| 433 |
+
for r in range(s_row, e_row + 1):
|
| 434 |
+
for c in range(min_col, max_col + 1):
|
| 435 |
+
if ws.cell(row=r, column=c).value is None:
|
| 436 |
+
blanks.append(f"{get_column_letter(c)}{r}")
|
| 437 |
+
|
| 438 |
+
return {
|
| 439 |
+
"check": "no_blanks",
|
| 440 |
+
"range": f"{check['sheet']}!{range_str}",
|
| 441 |
+
"passed": len(blanks) == 0,
|
| 442 |
+
"blank_count": len(blanks),
|
| 443 |
+
}
|
| 444 |
+
|
| 445 |
+
def _check_row_count_equals(self, wb: openpyxl.Workbook, check: dict) -> dict:
|
| 446 |
+
ws = wb[check["sheet"]]
|
| 447 |
+
expected = check["value"]
|
| 448 |
+
actual = 0
|
| 449 |
+
for row in ws.iter_rows(min_row=2):
|
| 450 |
+
if any(cell.value is not None for cell in row):
|
| 451 |
+
actual += 1
|
| 452 |
+
return {
|
| 453 |
+
"check": "row_count_equals",
|
| 454 |
+
"sheet": check["sheet"],
|
| 455 |
+
"passed": actual == expected,
|
| 456 |
+
"expected": expected,
|
| 457 |
+
"actual": actual,
|
| 458 |
+
}
|
| 459 |
+
|
| 460 |
+
def _check_all_dates_iso(self, wb: openpyxl.Workbook, check: dict) -> dict:
|
| 461 |
+
ws = wb[check["sheet"]]
|
| 462 |
+
col_letter = check["column"].upper()
|
| 463 |
+
col_idx = column_index_from_string(col_letter)
|
| 464 |
+
iso_re = re.compile(r"^\d{4}-\d{2}-\d{2}")
|
| 465 |
+
|
| 466 |
+
non_iso = []
|
| 467 |
+
for r in range(2, ws.max_row + 1):
|
| 468 |
+
val = ws.cell(row=r, column=col_idx).value
|
| 469 |
+
if val is None:
|
| 470 |
+
continue
|
| 471 |
+
if isinstance(val, (datetime, date)):
|
| 472 |
+
continue
|
| 473 |
+
if isinstance(val, str) and iso_re.match(val):
|
| 474 |
+
continue
|
| 475 |
+
non_iso.append(f"{col_letter}{r}: {val}")
|
| 476 |
+
|
| 477 |
+
return {
|
| 478 |
+
"check": "all_dates_iso_format",
|
| 479 |
+
"column": f"{check['sheet']}!{col_letter}",
|
| 480 |
+
"passed": len(non_iso) == 0,
|
| 481 |
+
"non_iso_count": len(non_iso),
|
| 482 |
+
}
|
| 483 |
+
|
| 484 |
+
def _check_constraint_satisfaction(self, wb: openpyxl.Workbook, check: dict) -> dict:
|
| 485 |
+
"""Evaluate domain constraints from a constraints sheet against an output sheet.
|
| 486 |
+
|
| 487 |
+
Constraints are read from prose text in column A of the constraints sheet.
|
| 488 |
+
Each row is a rule. The engine checks common patterns:
|
| 489 |
+
- "No employee works >N days"
|
| 490 |
+
- "Night→Morning gap required"
|
| 491 |
+
These are matched via regex and evaluated against the output grid.
|
| 492 |
+
"""
|
| 493 |
+
output_sheet = check["sheet"]
|
| 494 |
+
constraints_sheet = check.get("constraints_sheet", "Constraints")
|
| 495 |
+
|
| 496 |
+
if output_sheet not in wb.sheetnames:
|
| 497 |
+
return {"check": "constraint_satisfaction", "passed": False, "reason": f"Sheet '{output_sheet}' not found"}
|
| 498 |
+
if constraints_sheet not in wb.sheetnames:
|
| 499 |
+
return {"check": "constraint_satisfaction", "passed": False, "reason": f"Sheet '{constraints_sheet}' not found"}
|
| 500 |
+
|
| 501 |
+
ws_out = wb[output_sheet]
|
| 502 |
+
ws_con = wb[constraints_sheet]
|
| 503 |
+
|
| 504 |
+
constraints = []
|
| 505 |
+
for row in ws_con.iter_rows(min_col=1, max_col=1, values_only=True):
|
| 506 |
+
if row[0] and isinstance(row[0], str) and row[0].strip():
|
| 507 |
+
constraints.append(row[0].strip())
|
| 508 |
+
|
| 509 |
+
violations = []
|
| 510 |
+
for constraint_text in constraints:
|
| 511 |
+
violation = self._evaluate_constraint(ws_out, constraint_text)
|
| 512 |
+
if violation:
|
| 513 |
+
violations.append(violation)
|
| 514 |
+
|
| 515 |
+
return {
|
| 516 |
+
"check": "constraint_satisfaction",
|
| 517 |
+
"sheet": output_sheet,
|
| 518 |
+
"passed": len(violations) == 0,
|
| 519 |
+
"total_constraints": len(constraints),
|
| 520 |
+
"violations": violations,
|
| 521 |
+
}
|
| 522 |
+
|
| 523 |
+
def _evaluate_constraint(self, ws, constraint_text: str) -> Optional[str]:
|
| 524 |
+
"""Evaluate a single prose constraint against the output sheet.
|
| 525 |
+
Returns a violation description or None if satisfied."""
|
| 526 |
+
text_lower = constraint_text.lower()
|
| 527 |
+
|
| 528 |
+
max_days_match = re.search(r"no employee works?\s*>\s*(\d+)\s*days?", text_lower)
|
| 529 |
+
if max_days_match:
|
| 530 |
+
max_days = int(max_days_match.group(1))
|
| 531 |
+
for row in ws.iter_rows(min_row=2):
|
| 532 |
+
working_days = sum(
|
| 533 |
+
1 for cell in row[1:]
|
| 534 |
+
if cell.value is not None
|
| 535 |
+
and str(cell.value).upper().strip() not in ("X", "", "OFF")
|
| 536 |
+
)
|
| 537 |
+
emp = row[0].value
|
| 538 |
+
if working_days > max_days:
|
| 539 |
+
return f"{emp} works {working_days} days (max {max_days})"
|
| 540 |
+
return None
|
| 541 |
+
|
| 542 |
+
if "night" in text_lower and "morning" in text_lower and "gap" in text_lower:
|
| 543 |
+
for row in ws.iter_rows(min_row=2):
|
| 544 |
+
emp = row[0].value
|
| 545 |
+
shifts = [str(cell.value).upper().strip() if cell.value else "X" for cell in row[1:]]
|
| 546 |
+
for i in range(len(shifts) - 1):
|
| 547 |
+
if shifts[i] == "N" and shifts[i + 1] == "M":
|
| 548 |
+
return f"{emp} has Night→Morning on days {i + 1}→{i + 2}"
|
| 549 |
+
return None
|
| 550 |
+
|
| 551 |
+
return None
|
| 552 |
+
|
| 553 |
+
def _to_numeric(self, val: Any) -> Optional[float]:
|
| 554 |
+
if val is None:
|
| 555 |
+
return None
|
| 556 |
+
if isinstance(val, (int, float)):
|
| 557 |
+
return float(val)
|
| 558 |
+
if isinstance(val, str):
|
| 559 |
+
cleaned = val.replace(",", "").replace("$", "").replace("%", "").strip()
|
| 560 |
+
try:
|
| 561 |
+
return float(cleaned)
|
| 562 |
+
except (ValueError, TypeError):
|
| 563 |
+
return None
|
| 564 |
+
return None
|
spreadsheet.egg-info/PKG-INFO
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
Metadata-Version: 2.4
|
| 2 |
+
Name: spreadsheet
|
| 3 |
+
Version: 0.1.0
|
| 4 |
+
Requires-Python: >=3.11
|
| 5 |
+
Requires-Dist: openenv-core @ git+https://github.com/meta-pytorch/OpenEnv.git@v0.2.1
|
| 6 |
+
Requires-Dist: fastapi>=0.115.0
|
| 7 |
+
Requires-Dist: pydantic>=2.0.0
|
| 8 |
+
Requires-Dist: uvicorn[standard]>=0.24.0
|
| 9 |
+
Requires-Dist: fastmcp>=0.1.0
|
| 10 |
+
Requires-Dist: httpx>=0.25.0
|
| 11 |
+
Requires-Dist: openpyxl>=3.1.0
|
| 12 |
+
Requires-Dist: pandas>=2.0.0
|
| 13 |
+
Requires-Dist: formulas>=1.2.0
|
| 14 |
+
Provides-Extra: dev
|
| 15 |
+
Requires-Dist: pytest>=8.0.0; extra == "dev"
|
| 16 |
+
Requires-Dist: pytest-cov>=4.0.0; extra == "dev"
|
spreadsheet.egg-info/SOURCES.txt
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
README.md
|
| 2 |
+
__init__.py
|
| 3 |
+
client.py
|
| 4 |
+
generate_scenarios.py
|
| 5 |
+
models.py
|
| 6 |
+
pyproject.toml
|
| 7 |
+
./__init__.py
|
| 8 |
+
./client.py
|
| 9 |
+
./generate_scenarios.py
|
| 10 |
+
./models.py
|
| 11 |
+
server/__init__.py
|
| 12 |
+
server/app.py
|
| 13 |
+
server/formula_utils.py
|
| 14 |
+
server/scenario_loader.py
|
| 15 |
+
server/spreadsheet_environment.py
|
| 16 |
+
server/workbook_engine.py
|
| 17 |
+
spreadsheet.egg-info/PKG-INFO
|
| 18 |
+
spreadsheet.egg-info/SOURCES.txt
|
| 19 |
+
spreadsheet.egg-info/dependency_links.txt
|
| 20 |
+
spreadsheet.egg-info/entry_points.txt
|
| 21 |
+
spreadsheet.egg-info/requires.txt
|
| 22 |
+
spreadsheet.egg-info/top_level.txt
|
spreadsheet.egg-info/dependency_links.txt
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
|
spreadsheet.egg-info/entry_points.txt
ADDED
|
@@ -0,0 +1,2 @@
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[console_scripts]
|
| 2 |
+
server = spreadsheet.server.app:main
|
spreadsheet.egg-info/requires.txt
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
openenv-core @ git+https://github.com/meta-pytorch/OpenEnv.git@v0.2.1
|
| 2 |
+
fastapi>=0.115.0
|
| 3 |
+
pydantic>=2.0.0
|
| 4 |
+
uvicorn[standard]>=0.24.0
|
| 5 |
+
fastmcp>=0.1.0
|
| 6 |
+
httpx>=0.25.0
|
| 7 |
+
openpyxl>=3.1.0
|
| 8 |
+
pandas>=2.0.0
|
| 9 |
+
formulas>=1.2.0
|
| 10 |
+
|
| 11 |
+
[dev]
|
| 12 |
+
pytest>=8.0.0
|
| 13 |
+
pytest-cov>=4.0.0
|
spreadsheet.egg-info/top_level.txt
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
spreadsheet
|
uv.lock
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
workbooks/fixtures/.gitkeep
ADDED
|
File without changes
|
workbooks/fixtures/037858b5-3d0e-4714-8640-2dea23fc3a18_multi_currency_reconciliation.xlsx
ADDED
|
Binary file (8 kB). View file
|
|
|
workbooks/fixtures/1333ba32-7957-4f7f-b310-6a9ba0e718bd_data_pivot_reshape.xlsx
ADDED
|
Binary file (9.93 kB). View file
|
|
|
workbooks/fixtures/15123e53-9510-48d4-ae1a-a01556145b8e_employee_bonus_calculation.xlsx
ADDED
|
Binary file (9.15 kB). View file
|
|
|
workbooks/fixtures/158cebfc-4813-49c4-bd54-fceef44c4860_employee_schedule_grid.xlsx
ADDED
|
Binary file (7.38 kB). View file
|
|
|
workbooks/fixtures/19d8a671-1769-45aa-af51-39d12e81d45c_multi_currency_reconciliation.xlsx
ADDED
|
Binary file (8 kB). View file
|
|
|
workbooks/fixtures/30f43287-34a1-4620-a9ae-3d982705a5e5_bank_reconciliation.xlsx
ADDED
|
Binary file (10.8 kB). View file
|
|
|
workbooks/fixtures/45bb730b-c042-491e-9f7d-ff9ff3de25a6_cascading_formula_errors.xlsx
ADDED
|
Binary file (6.49 kB). View file
|
|
|
workbooks/fixtures/5a43518a-7b4c-4e49-a2e3-ae0e550f5351_multi_department_budget.xlsx
ADDED
|
Binary file (9.1 kB). View file
|
|
|