Upload folder using huggingface_hub

Dockerfile

@@ -0,0 +1,81 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+# Multi-stage build using openenv-base
+# This Dockerfile is flexible and works for both:
+# - In-repo environments (with local OpenEnv sources)
+# - Standalone environments (with openenv from PyPI/Git)
+# The build script (openenv build) handles context detection and sets appropriate build args.
+
+ARG BASE_IMAGE=ghcr.io/meta-pytorch/openenv-base:latest
+FROM ${BASE_IMAGE} AS builder
+
+WORKDIR /app
+
+# Ensure git is available (required for installing dependencies from VCS)
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends git && \
+    rm -rf /var/lib/apt/lists/*
+
+# Build argument to control whether we're building standalone or in-repo
+ARG BUILD_MODE=in-repo
+ARG ENV_NAME=ask_answer_env
+
+# Copy environment code (always at root of build context)
+COPY . /app/env
+
+# For in-repo builds, openenv is already vendored in the build context
+# For standalone builds, openenv will be installed via pyproject.toml
+WORKDIR /app/env
+
+# Ensure uv is available (for local builds where base image lacks it)
+RUN if ! command -v uv >/dev/null 2>&1; then \
+        curl -LsSf https://astral.sh/uv/install.sh | sh && \
+        mv /root/.local/bin/uv /usr/local/bin/uv && \
+        mv /root/.local/bin/uvx /usr/local/bin/uvx; \
+    fi
+    
+# Install dependencies using uv sync
+# If uv.lock exists, use it; otherwise resolve on the fly
+RUN --mount=type=cache,target=/root/.cache/uv \
+    if [ -f uv.lock ]; then \
+        uv sync --frozen --no-install-project --no-editable; \
+    else \
+        uv sync --no-install-project --no-editable; \
+    fi
+
+RUN --mount=type=cache,target=/root/.cache/uv \
+    if [ -f uv.lock ]; then \
+        uv sync --frozen --no-editable; \
+    else \
+        uv sync --no-editable; \
+    fi
+
+# Final runtime stage
+FROM ${BASE_IMAGE}
+
+WORKDIR /app
+
+# Copy the virtual environment from builder
+COPY --from=builder /app/env/.venv /app/.venv
+
+# Copy the environment code
+COPY --from=builder /app/env /app/env
+
+# Set PATH to use the virtual environment
+ENV PATH="/app/.venv/bin:$PATH"
+
+# Set PYTHONPATH so imports work correctly
+ENV PYTHONPATH="/app/env:$PYTHONPATH"
+
+# Health check
+HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \
+    CMD curl -f http://localhost:8000/health || exit 1
+
+# Run the FastAPI server
+# The module path is constructed to work with the /app/env structure
+ENV ENABLE_WEB_INTERFACE=true
+CMD ["sh", "-c", "cd /app/env && uvicorn server.app:app --host 0.0.0.0 --port 8000"]

README.md

@@ -1,12 +1,204 @@
 ---
 title: Ask Answer Env
-emoji: ⚡
-colorFrom: green
-colorTo: purple
+emoji: 🎯
+colorFrom: blue
+colorTo: green
 sdk: docker
-pinned: false
-license: mit
-short_description: OpenEnv for training to decide between asking & answering
+app_port: 8000
+base_path: /web
+tags:
+  - openenv
+  - rl
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# Ask Answer Env (v1)
+
+A deterministic OpenEnv environment for training RL agents to decide between **asking clarifying questions** or **answering early** under budget constraints.
+
+## Overview
+
+The agent receives a user prompt ("Plan a short trip for me.") and must discover hidden slot values by asking questions before providing a final answer. With only **3 steps** and **4 slots** (3 core + 1 distractor), the agent must prioritize which questions to ask.
+
+**Key design goals:**
+- No ML, no NLP — just structured interaction + delayed reward
+- Deterministic given a seed
+- Budget constraints force non-trivial tradeoffs (can only ask 2 of 4 slots)
+- Graded reward structure (partial credit for correct slots)
+
+## Hidden State
+
+At each episode reset, the environment samples (with seeded RNG):
+- `city` ∈ `["Paris", "Rome", "Tokyo", "Goa"]` (core)
+- `date` ∈ `["next_weekend", "mid_feb", "march"]` (core)
+- `budget` ∈ `["low", "mid", "high"]` (core)
+- `style` ∈ `["relax", "adventure", "food"]` (distractor)
+
+The agent cannot see hidden values unless it asks.
+
+## Action Space
+
+**ASK** — reveal a slot:
+```python
+AskAnswerAction(type="ask", slot="city")  # or "date", "budget", "style"
+```
+
+**ANSWER** — end episode with guesses:
+```python
+AskAnswerAction(type="answer", city="Paris", date="mid_feb", budget="high", style="relax")
+```
+
+## Observation
+
+```python
+{
+    "prompt": "Plan a short trip for me.",
+    "known": {
+        "city": None | str,
+        "date": None | str,
+        "budget": None | str,
+        "style": None | str
+    },
+    "steps_left": int,  # starts at 3
+    "core_correct_count": int | None  # populated after ANSWER (0-3)
+}
+```
+
+## Rewards (v1 - Graded Scoring)
+
+| Event | Reward |
+|-------|--------|
+| Step penalty (always) | -0.05 |
+| ASK unknown slot | +0.10 |
+| ASK already-known slot | -0.20 |
+| City correct | +0.40 |
+| Date correct | +0.40 |
+| Budget correct | +0.40 |
+| Style correct (bonus) | +0.10 |
+| All 3 core slots correct (bonus) | +0.20 |
+| Any core slot wrong (penalty) | -0.60 |
+
+**Oracle reward (theoretical max):** +1.45 (knows everything, answers perfectly in 1 step)
+
+## Baseline Results
+
+```
+==========================================================================================
+RESULTS SUMMARY (200 episodes each)
+==========================================================================================
+Baseline                   Mean     Std    Pos%   Core%  AvgCore
+------------------------------------------------------------------------------------------
+Oracle (theoretical)     +1.450   0.000   100%   100%    3.00/3
+B: city+budget           +0.634   0.560   100%    32%    2.32/3
+A: city+date             +0.604   0.547   100%    30%    2.29/3
+C: style+city (trap)     +0.284   0.483    50%    11%    1.61/3
+Random                   -0.134   0.530    30%     6%    1.08/3
+------------------------------------------------------------------------------------------
+
+Column legend:
+  Mean    = mean total reward
+  Pos%    = positive_return_rate (% episodes with reward > 0)
+  Core%   = core_success_rate (% episodes with all 3 core slots correct)
+  AvgCore = avg_core_correct (mean # of core slots correct, out of 3)
+```
+
+**Key insights:**
+- A/B strategies (ask 2 core slots) achieve ~100% positive return
+- C strategy (wastes a question on style distractor) drops to ~50%
+- Random baseline performs poorly (~30% positive return)
+- Core success rate ~30% for A/B matches expected 1/3 (guessing 1 slot)
+
+## Quick Start
+
+### Build Docker Image
+
+```bash
+# For local use (root Dockerfile used by HF Spaces)
+docker build -t ask_answer_env-env:latest .
+
+# Or use server/Dockerfile (equivalent)
+docker build -t ask_answer_env-env:latest -f server/Dockerfile .
+```
+
+### Run Baseline Tests
+
+```bash
+uv run python exp.py
+```
+
+### Example Usage
+
+```python
+from ask_answer_env import AskAnswerEnv, AskAnswerAction
+
+client = AskAnswerEnv.from_docker_image("ask_answer_env-env:latest")
+try:
+    result = client.reset(seed=42)
+    print(f"Steps left: {result.observation.steps_left}")  # 3
+
+    # Ask about city (step 1)
+    result = client.step(AskAnswerAction(type="ask", slot="city"))
+    print(f"City: {result.observation.known.city}")
+
+    # Ask about date (step 2)
+    result = client.step(AskAnswerAction(type="ask", slot="date"))
+    print(f"Date: {result.observation.known.date}")
+
+    # Must answer now (step 3) - guess budget
+    known = result.observation.known
+    result = client.step(AskAnswerAction(
+        type="answer",
+        city=known.city,
+        date=known.date,
+        budget="mid",  # guess
+    ))
+    print(f"Final reward: {result.reward}")
+    print(f"Core correct: {result.observation.core_correct_count}/3")
+finally:
+    client.close()
+```
+
+## Testing (`exp.py`)
+
+The `exp.py` script contains:
+
+### 1. Determinism Tests
+Verifies same seed → identical trajectories and rewards.
+
+### 2. Seed Sensitivity Test
+Confirms different seeds produce different hidden states.
+
+### 3. Baseline Comparison
+Runs 5 strategies over 200 episodes each:
+- **Oracle**: Theoretical upper bound (knows hidden state)
+- **A: city+date**: Ask city, ask date, guess budget
+- **B: city+budget**: Ask city, ask budget, guess date
+- **C: style+city (trap)**: Wastes a question on distractor
+- **Random**: Random ask/answer decisions
+
+### 4. Ordering Verification
+Confirms: Oracle > A ≈ B >> C > Random
+
+## Project Structure
+
+```
+ask_answer_env/
+├── __init__.py           # Module exports
+├── models.py             # AskAnswerAction, AskAnswerObservation, KnownSlots
+├── client.py             # AskAnswerEnv client (WebSocket)
+├── exp.py                # Baseline strategies + acceptance tests
+├── Dockerfile            # Root Dockerfile (for HF Spaces)
+├── server/
+│   ├── ask_answer_env_environment.py  # Core environment logic
+│   ├── app.py            # FastAPI server
+│   └── Dockerfile
+├── openenv.yaml          # OpenEnv manifest
+├── pyproject.toml        # Dependencies
+└── uv.lock               # Locked deps
+```
+
+## Episode Rules
+
+- `max_steps = 3`
+- Episode ends when agent sends ANSWER or steps run out
+- Auto-fail (steps exhausted) gives -1.0 reward
+- With 3 steps, agent can ask at most 2 slots before forced to answer/fail

__init__.py

@@ -0,0 +1,17 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+"""Ask Answer Env Environment."""
+
+from .client import AskAnswerEnv
+from .models import AskAnswerAction, AskAnswerObservation, KnownSlots
+
+__all__ = [
+    "AskAnswerAction",
+    "AskAnswerObservation",
+    "KnownSlots",
+    "AskAnswerEnv",
+]

client.py

@@ -0,0 +1,123 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+"""Ask Answer Env Environment Client."""
+
+from typing import Dict
+
+from openenv.core.client_types import StepResult
+from openenv.core.env_server.types import State
+from openenv.core import EnvClient
+
+from .models import AskAnswerAction, AskAnswerObservation, KnownSlots
+
+
+class AskAnswerEnv(
+    EnvClient[AskAnswerAction, AskAnswerObservation, State]
+):
+    """
+    Client for the Ask Answer Env Environment.
+
+    A slot-filling environment where agents must decide between asking
+    clarifying questions or answering early to maximize reward.
+
+    Example:
+        >>> with AskAnswerEnv(base_url="http://localhost:8000") as client:
+        ...     result = client.reset(seed=42)
+        ...     print(result.observation.prompt)
+        ...     print(result.observation.known)
+        ...
+        ...     # Ask about city
+        ...     result = client.step(AskAnswerAction(type="ask", slot="city"))
+        ...     print(f"City: {result.observation.known.city}")
+        ...
+        ...     # Answer with known values
+        ...     result = client.step(AskAnswerAction(
+        ...         type="answer",
+        ...         city=result.observation.known.city,
+        ...         date="mid_feb",
+        ...         budget="high"
+        ...     ))
+        ...     print(f"Reward: {result.reward}, Done: {result.done}")
+
+    Example with Docker:
+        >>> client = AskAnswerEnv.from_docker_image("ask_answer_env-env:latest")
+        >>> try:
+        ...     result = client.reset(seed=42)
+        ...     # ... interact with environment
+        ... finally:
+        ...     client.close()
+    """
+
+    def _step_payload(self, action: AskAnswerAction) -> Dict:
+        """
+        Convert AskAnswerAction to JSON payload for step message.
+
+        Args:
+            action: AskAnswerAction instance
+
+        Returns:
+            Dictionary representation suitable for JSON encoding
+        """
+        payload = {"type": action.type}
+        if action.type == "ask":
+            payload["slot"] = action.slot
+        else:  # answer
+            payload["city"] = action.city
+            payload["date"] = action.date
+            payload["budget"] = action.budget
+            payload["style"] = action.style
+        return payload
+
+    def _parse_result(self, payload: Dict) -> StepResult[AskAnswerObservation]:
+        """
+        Parse server response into StepResult[AskAnswerObservation].
+
+        Args:
+            payload: JSON response data from server
+
+        Returns:
+            StepResult with AskAnswerObservation
+        """
+        obs_data = payload.get("observation", {})
+        known_data = obs_data.get("known", {})
+        
+        known = KnownSlots(
+            city=known_data.get("city"),
+            date=known_data.get("date"),
+            budget=known_data.get("budget"),
+            style=known_data.get("style"),
+        )
+        
+        observation = AskAnswerObservation(
+            prompt=obs_data.get("prompt", "Plan a short trip for me."),
+            known=known,
+            steps_left=obs_data.get("steps_left", 0),
+            done=payload.get("done", False),
+            reward=payload.get("reward"),
+            core_correct_count=obs_data.get("core_correct_count"),
+        )
+
+        return StepResult(
+            observation=observation,
+            reward=payload.get("reward"),
+            done=payload.get("done", False),
+        )
+
+    def _parse_state(self, payload: Dict) -> State:
+        """
+        Parse server response into State object.
+
+        Args:
+            payload: JSON response from state request
+
+        Returns:
+            State object with episode_id and step_count
+        """
+        return State(
+            episode_id=payload.get("episode_id"),
+            step_count=payload.get("step_count", 0),
+        )

exp.py

@@ -0,0 +1,463 @@
+"""
+Baseline agents for the Ask Answer Env environment (v1).
+
+Tests different ask-vs-act strategies under budget constraints (MAX_STEPS=3).
+With only 3 steps, agents can ask at most 2 slots before being forced to answer,
+creating a non-trivial tradeoff between information gathering and guessing.
+
+Baselines:
+- A: city+date (ask city, ask date, guess budget)
+- B: city+budget (ask city, ask budget, guess date)
+- C: style+city (trap: wastes a question on distractor)
+- Random: random actions
+- Oracle: knows hidden state, answers immediately (upper bound)
+"""
+
+import random
+from dataclasses import dataclass
+from typing import Callable, List, Optional, Tuple
+
+from ask_answer_env import AskAnswerEnv, AskAnswerAction, KnownSlots
+
+
+# Type aliases
+HiddenTuple = Tuple[str, str, str, str]  # (city, date, budget, style)
+StrategyFn = Callable[[KnownSlots, int, Optional[HiddenTuple]], AskAnswerAction]
+
+# Default guesses when slot is unknown
+DEFAULT_CITY = "Paris"
+DEFAULT_DATE = "mid_feb"
+DEFAULT_BUDGET = "mid"
+DEFAULT_STYLE = "relax"
+
+# Valid slot values (for random baseline)
+CITIES = ["Paris", "Rome", "Tokyo", "Goa"]
+DATES = ["next_weekend", "mid_feb", "march"]
+BUDGETS = ["low", "mid", "high"]
+STYLES = ["relax", "adventure", "food"]
+
+
+@dataclass
+class EpisodeResult:
+    """Result of a single episode."""
+    total_reward: float
+    revealed: HiddenTuple
+    steps_taken: int
+    core_correct_count: int  # 0-3: how many core slots were correct
+    core_all_correct: bool   # True if all 3 core slots correct
+
+
+# =============================================================================
+# Strategy Functions
+# =============================================================================
+
+def strategy_city_date(known: KnownSlots, steps_left: int, hidden: Optional[HiddenTuple] = None) -> AskAnswerAction:
+    """
+    Strategy A: Ask city, ask date, then answer (guess budget).
+    
+    Expected behavior with MAX_STEPS=3:
+    - Step 1: ASK city
+    - Step 2: ASK date
+    - Step 3: ANSWER with known city+date, guess budget
+    """
+    if known.city is None:
+        return AskAnswerAction(type="ask", slot="city")
+    elif known.date is None:
+        return AskAnswerAction(type="ask", slot="date")
+    else:
+        return AskAnswerAction(
+            type="answer",
+            city=known.city,
+            date=known.date,
+            budget=known.budget if known.budget else DEFAULT_BUDGET,
+            style=known.style,  # None if not asked
+        )
+
+
+def strategy_city_budget(known: KnownSlots, steps_left: int, hidden: Optional[HiddenTuple] = None) -> AskAnswerAction:
+    """
+    Strategy B: Ask city, ask budget, then answer (guess date).
+    
+    Expected behavior with MAX_STEPS=3:
+    - Step 1: ASK city
+    - Step 2: ASK budget
+    - Step 3: ANSWER with known city+budget, guess date
+    """
+    if known.city is None:
+        return AskAnswerAction(type="ask", slot="city")
+    elif known.budget is None:
+        return AskAnswerAction(type="ask", slot="budget")
+    else:
+        return AskAnswerAction(
+            type="answer",
+            city=known.city,
+            date=known.date if known.date else DEFAULT_DATE,
+            budget=known.budget,
+            style=known.style,
+        )
+
+
+def strategy_style_city(known: KnownSlots, steps_left: int, hidden: Optional[HiddenTuple] = None) -> AskAnswerAction:
+    """
+    Strategy C (TRAP): Ask style first, then city, guess date+budget.
+    
+    This is a BAD strategy because:
+    - Style only gives +0.1 bonus (vs +0.4 for core slots)
+    - Wastes a question on a low-value distractor
+    - Must guess 2 core slots instead of 1
+    
+    Expected behavior with MAX_STEPS=3:
+    - Step 1: ASK style (bad choice!)
+    - Step 2: ASK city
+    - Step 3: ANSWER with known style+city, guess date+budget
+    """
+    if known.style is None:
+        return AskAnswerAction(type="ask", slot="style")
+    elif known.city is None:
+        return AskAnswerAction(type="ask", slot="city")
+    else:
+        return AskAnswerAction(
+            type="answer",
+            city=known.city,
+            date=known.date if known.date else DEFAULT_DATE,
+            budget=known.budget if known.budget else DEFAULT_BUDGET,
+            style=known.style,
+        )
+
+
+def strategy_random(known: KnownSlots, steps_left: int, hidden: Optional[HiddenTuple] = None) -> AskAnswerAction:
+    """
+    Random baseline: randomly ask or answer with random values.
+    
+    50% chance to ask a random unknown slot, 50% chance to answer.
+    If no unknown slots, always answer.
+    """
+    unknown_slots = []
+    if known.city is None:
+        unknown_slots.append("city")
+    if known.date is None:
+        unknown_slots.append("date")
+    if known.budget is None:
+        unknown_slots.append("budget")
+    if known.style is None:
+        unknown_slots.append("style")
+    
+    # If we have unknown slots and coin flip says ask
+    if unknown_slots and random.random() < 0.5:
+        slot = random.choice(unknown_slots)
+        return AskAnswerAction(type="ask", slot=slot)
+    
+    # Otherwise answer with random guesses for unknown slots
+    return AskAnswerAction(
+        type="answer",
+        city=known.city if known.city else random.choice(CITIES),
+        date=known.date if known.date else random.choice(DATES),
+        budget=known.budget if known.budget else random.choice(BUDGETS),
+        style=known.style if known.style else random.choice(STYLES),
+    )
+
+
+def strategy_oracle(known: KnownSlots, steps_left: int, hidden: Optional[HiddenTuple] = None) -> AskAnswerAction:
+    """
+    Oracle baseline: knows hidden state, answers perfectly in 1 step.
+    
+    This is the THEORETICAL UPPER BOUND.
+    
+    In practice, this strategy function is NOT used because the server
+    doesn't expose hidden state to the client. Instead, we hardcode
+    the oracle's reward as 1.45 in run_baseline_test().
+    
+    Reward breakdown:
+        -0.05 (step) + 0.4×3 (core) + 0.1 (style) + 0.2 (bonus) = +1.45
+    """
+    if hidden is None:
+        raise ValueError("Oracle strategy requires hidden state")
+    
+    city, date, budget, style = hidden
+    return AskAnswerAction(
+        type="answer",
+        city=city,
+        date=date,
+        budget=budget,
+        style=style,
+    )
+
+
+# =============================================================================
+# Episode Runner
+# =============================================================================
+
+def run_episode(
+    client: AskAnswerEnv,
+    strategy: StrategyFn,
+    seed: int = 42,
+    hidden: Optional[HiddenTuple] = None,
+    verbose: bool = False,
+) -> EpisodeResult:
+    """
+    Run a single episode with the given strategy.
+    
+    Args:
+        client: AskAnswerEnv client instance
+        strategy: Function that takes (known, steps_left, hidden) and returns action
+        seed: Random seed for reproducibility
+        hidden: Hidden state tuple (required for oracle strategy)
+        verbose: Whether to print step-by-step info
+    
+    Returns:
+        EpisodeResult with total_reward, revealed slots, and steps taken
+    """
+    result = client.reset(seed=seed)
+    total_reward = 0.0
+    steps = 0
+    
+    if verbose:
+        print(f"=== Episode Start (seed={seed}) ===")
+        print(f"Steps left: {result.observation.steps_left}")
+    
+    while not result.done:
+        obs = result.observation
+        action = strategy(obs.known, obs.steps_left, hidden)
+        
+        result = client.step(action)
+        total_reward += result.reward
+        steps += 1
+        
+        if verbose:
+            if action.type == "ask":
+                slot_val = getattr(result.observation.known, action.slot)
+                print(f"  Step {steps}: ASK {action.slot} -> {slot_val}, reward={result.reward:+.2f}")
+            else:
+                print(f"  Step {steps}: ANSWER city={action.city}, date={action.date}, "
+                      f"budget={action.budget}, style={action.style}, reward={result.reward:+.2f}")
+    
+    final = result.observation.known
+    revealed = (final.city, final.date, final.budget, final.style)
+    
+    # Extract correctness info (available when done=True after ANSWER)
+    core_correct_count = result.observation.core_correct_count or 0
+    core_all_correct = core_correct_count == 3
+    
+    if verbose:
+        print(f"  Total reward: {total_reward:+.2f}")
+        print(f"  Core correct: {core_correct_count}/3")
+        print()
+    
+    return EpisodeResult(
+        total_reward=total_reward,
+        revealed=revealed,
+        steps_taken=steps,
+        core_correct_count=core_correct_count,
+        core_all_correct=core_all_correct,
+    )
+
+
+# =============================================================================
+# Acceptance Tests
+# =============================================================================
+
+@dataclass
+class BaselineStats:
+    """Statistics for a baseline over multiple episodes."""
+    name: str
+    mean_reward: float
+    std_reward: float
+    positive_return_rate: float  # % of episodes with reward > 0
+    core_success_rate: float     # % of episodes with all 3 core slots correct
+    avg_core_correct: float      # average number of core slots correct (0-3)
+
+
+def run_baseline_test(
+    client: AskAnswerEnv,
+    name: str,
+    strategy: StrategyFn,
+    num_episodes: int = 200,
+    needs_oracle: bool = False,
+) -> BaselineStats:
+    """
+    Run multiple episodes with a strategy and compute statistics.
+    
+    Args:
+        client: AskAnswerEnv client instance
+        name: Name of the baseline for logging
+        strategy: Strategy function
+        num_episodes: Number of episodes to run
+        needs_oracle: If True, use theoretical oracle values
+    
+    Returns:
+        BaselineStats with all metrics
+    """
+    if needs_oracle:
+        # Oracle is a THEORETICAL upper bound - knows hidden state,
+        # answers perfectly in 1 step.
+        #
+        # Reward: -0.05 + 0.4×3 + 0.1 + 0.2 = +1.45
+        # Core correct: 3/3 always
+        return BaselineStats(
+            name=name,
+            mean_reward=1.45,
+            std_reward=0.0,
+            positive_return_rate=1.0,
+            core_success_rate=1.0,
+            avg_core_correct=3.0,
+        )
+    
+    results: List[EpisodeResult] = []
+    
+    for seed in range(num_episodes):
+        result = run_episode(client, strategy, seed=seed)
+        results.append(result)
+    
+    rewards = [r.total_reward for r in results]
+    mean_reward = sum(rewards) / len(rewards)
+    variance = sum((r - mean_reward) ** 2 for r in rewards) / len(rewards)
+    std_reward = variance ** 0.5
+    
+    positive_return_rate = sum(1 for r in rewards if r > 0) / len(rewards)
+    core_success_rate = sum(1 for r in results if r.core_all_correct) / len(results)
+    avg_core_correct = sum(r.core_correct_count for r in results) / len(results)
+    
+    return BaselineStats(
+        name=name,
+        mean_reward=mean_reward,
+        std_reward=std_reward,
+        positive_return_rate=positive_return_rate,
+        core_success_rate=core_success_rate,
+        avg_core_correct=avg_core_correct,
+    )
+
+
+def run_acceptance_tests(client: AskAnswerEnv, num_episodes: int = 200) -> bool:
+    """
+    Run all baseline tests and print results table.
+    
+    Expected ordering:
+    Oracle > A ≈ B >> C > Random
+    """
+    print(f"\nRunning {num_episodes} episodes per baseline...\n")
+    
+    baselines = [
+        ("Oracle (theoretical)", None, True),
+        ("A: city+date", strategy_city_date, False),
+        ("B: city+budget", strategy_city_budget, False),
+        ("C: style+city (trap)", strategy_style_city, False),
+        ("Random", strategy_random, False),
+    ]
+    
+    all_stats: List[BaselineStats] = []
+    for name, strategy, is_oracle in baselines:
+        stats = run_baseline_test(client, name, strategy, num_episodes, needs_oracle=is_oracle)
+        all_stats.append(stats)
+        print(f"  {name}: mean={stats.mean_reward:+.3f}, core_success={stats.core_success_rate:.1%}")
+    
+    # Print results table
+    print("\n" + "=" * 90)
+    print("RESULTS SUMMARY")
+    print("=" * 90)
+    header = f"{'Baseline':<22} {'Mean':>8} {'Std':>7} {'Pos%':>7} {'Core%':>7} {'AvgCore':>8}"
+    print(header)
+    print("-" * 90)
+    
+    for s in sorted(all_stats, key=lambda x: -x.mean_reward):
+        print(f"{s.name:<22} {s.mean_reward:>+8.3f} {s.std_reward:>7.3f} "
+              f"{s.positive_return_rate:>6.0%} {s.core_success_rate:>6.0%} "
+              f"{s.avg_core_correct:>7.2f}/3")
+    print("-" * 90)
+    
+    print("\nColumn legend:")
+    print("  Mean    = mean total reward")
+    print("  Std     = standard deviation of reward")
+    print("  Pos%    = positive_return_rate (% episodes with reward > 0)")
+    print("  Core%   = core_success_rate (% episodes with all 3 core slots correct)")
+    print("  AvgCore = avg_core_correct (mean # of core slots correct, out of 3)")
+    
+    # Verify expected ordering
+    result_dict = {s.name: s.mean_reward for s in all_stats}
+    
+    checks = [
+        ("Oracle > A", result_dict["Oracle (theoretical)"] > result_dict["A: city+date"]),
+        ("A ≈ B", abs(result_dict["A: city+date"] - result_dict["B: city+budget"]) < 0.1),
+        ("A > C", result_dict["A: city+date"] > result_dict["C: style+city (trap)"]),
+        ("C > Random", result_dict["C: style+city (trap)"] > result_dict["Random"]),
+    ]
+    
+    print("\nExpected ordering checks:")
+    all_passed = True
+    for check_name, passed in checks:
+        status = "PASS" if passed else "FAIL"
+        print(f"  {check_name}: {status}")
+        if not passed:
+            all_passed = False
+    
+    return all_passed
+
+
+# =============================================================================
+# Determinism Tests (kept from v0)
+# =============================================================================
+
+def test_determinism(client: AskAnswerEnv, seed: int = 42, runs: int = 3) -> bool:
+    """Test that the same seed produces identical trajectories."""
+    trajectories = []
+    for _ in range(runs):
+        result = run_episode(client, strategy_city_date, seed=seed)
+        trajectories.append((result.total_reward, result.revealed))
+    
+    rewards = [t[0] for t in trajectories]
+    revealed = [t[1] for t in trajectories]
+    
+    identical = len(set(revealed)) == 1 and len(set(rewards)) == 1
+    print(f"Determinism (seed={seed}): {revealed[0]} x{runs}, identical={identical}")
+    return identical
+
+
+def test_seed_sensitivity(client: AskAnswerEnv, num_seeds: int = 20) -> bool:
+    """Verify different seeds produce different hidden states."""
+    unique = set()
+    for seed in range(num_seeds):
+        result = run_episode(client, strategy_city_date, seed=seed)
+        unique.add(result.revealed)
+    
+    # Max possible: 4 * 3 * 3 * 3 = 108 (with style)
+    print(f"Seed sensitivity: {len(unique)} unique tuples from {num_seeds} seeds")
+    return len(unique) > 1
+
+
+# =============================================================================
+# Main
+# =============================================================================
+
+if __name__ == "__main__":
+    client = AskAnswerEnv.from_docker_image("ask_answer_env-env:latest")
+    try:
+        print("=" * 60)
+        print("ASK-ANSWER ENV v1 ACCEPTANCE TESTS")
+        print("=" * 60)
+        
+        # Quick determinism check
+        print("\n1. DETERMINISM TESTS")
+        print("-" * 40)
+        test_determinism(client, seed=42)
+        test_determinism(client, seed=123)
+        test_seed_sensitivity(client)
+        
+        # Run a single verbose episode to show behavior
+        print("\n2. EXAMPLE EPISODE (Strategy A: city+date)")
+        print("-" * 40)
+        run_episode(client, strategy_city_date, seed=42, verbose=True)
+        
+        print("\n3. EXAMPLE EPISODE (Strategy C: style+city - TRAP)")
+        print("-" * 40)
+        run_episode(client, strategy_style_city, seed=42, verbose=True)
+        
+        # Full acceptance tests
+        print("\n4. BASELINE COMPARISON")
+        print("-" * 40)
+        passed = run_acceptance_tests(client, num_episodes=200)
+        
+        print("\n" + "=" * 60)
+        print(f"ALL TESTS: {'PASSED' if passed else 'FAILED'}")
+        print("=" * 60)
+        
+    finally:
+        client.close()

models.py

@@ -0,0 +1,59 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+"""
+Data models for the Ask Answer Env Environment.
+
+A slot-filling environment where agents must decide between asking
+clarifying questions or answering early.
+"""
+
+from typing import Literal, Optional
+
+from pydantic import BaseModel, Field
+
+from openenv.core.env_server.types import Action, Observation
+
+
+class AskAnswerAction(Action):
+    """
+    Action for the Ask Answer Env environment.
+    
+    Two action types:
+    - "ask": Ask about a specific slot (city, date, budget, or style)
+    - "answer": Provide final answer with slot values
+    """
+
+    type: Literal["ask", "answer"] = Field(..., description="Action type: 'ask' or 'answer'")
+    slot: Optional[Literal["city", "date", "budget", "style"]] = Field(
+        default=None, description="Slot to ask about (only for 'ask' type)"
+    )
+    city: Optional[str] = Field(default=None, description="City answer (only for 'answer' type)")
+    date: Optional[str] = Field(default=None, description="Date answer (only for 'answer' type)")
+    budget: Optional[str] = Field(default=None, description="Budget answer (only for 'answer' type)")
+    style: Optional[str] = Field(default=None, description="Style answer (only for 'answer' type)")
+
+
+class KnownSlots(BaseModel):
+    """Slots that have been revealed to the agent."""
+
+    city: Optional[str] = Field(default=None, description="Known city value")
+    date: Optional[str] = Field(default=None, description="Known date value")
+    budget: Optional[str] = Field(default=None, description="Known budget value")
+    style: Optional[str] = Field(default=None, description="Known style value")
+
+
+class AskAnswerObservation(Observation):
+    """Observation from the Ask Answer Env environment."""
+
+    prompt: str = Field(default="Plan a short trip for me.", description="The user prompt")
+    known: KnownSlots = Field(default_factory=KnownSlots, description="Currently known slot values")
+    steps_left: int = Field(default=3, description="Number of steps remaining")
+    
+    # Correctness info (populated when done=True after ANSWER action)
+    core_correct_count: Optional[int] = Field(
+        default=None, description="Number of core slots correct (0-3), None if not answered yet"
+    )

openenv.yaml

@@ -0,0 +1,7 @@
+spec_version: 1
+name: ask_answer_env
+type: space
+runtime: fastapi
+app: server.app:app
+port: 8000
+

openenv_ask_answer_env.egg-info/PKG-INFO

@@ -0,0 +1,9 @@
+Metadata-Version: 2.4
+Name: openenv-ask_answer_env
+Version: 0.1.0
+Summary: Ask Answer Env environment for OpenEnv
+Requires-Python: >=3.10
+Requires-Dist: openenv-core[core]>=0.2.1
+Provides-Extra: dev
+Requires-Dist: pytest>=8.0.0; extra == "dev"
+Requires-Dist: pytest-cov>=4.0.0; extra == "dev"

openenv_ask_answer_env.egg-info/SOURCES.txt

@@ -0,0 +1,18 @@
+README.md
+__init__.py
+client.py
+models.py
+pyproject.toml
+./__init__.py
+./client.py
+./exp.py
+./models.py
+openenv_ask_answer_env.egg-info/PKG-INFO
+openenv_ask_answer_env.egg-info/SOURCES.txt
+openenv_ask_answer_env.egg-info/dependency_links.txt
+openenv_ask_answer_env.egg-info/entry_points.txt
+openenv_ask_answer_env.egg-info/requires.txt
+openenv_ask_answer_env.egg-info/top_level.txt
+server/__init__.py
+server/app.py
+server/ask_answer_env_environment.py

openenv_ask_answer_env.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

openenv_ask_answer_env.egg-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+server = ask_answer_env.server.app:main

openenv_ask_answer_env.egg-info/requires.txt

@@ -0,0 +1,5 @@
+openenv-core[core]>=0.2.1
+
+[dev]
+pytest>=8.0.0
+pytest-cov>=4.0.0

openenv_ask_answer_env.egg-info/top_level.txt

@@ -0,0 +1 @@
+ask_answer_env

pyproject.toml

@@ -0,0 +1,45 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+[build-system]
+requires = ["setuptools>=45", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "openenv-ask_answer_env"
+version = "0.1.0"
+description = "Ask Answer Env environment for OpenEnv"
+requires-python = ">=3.10"
+dependencies = [
+    # Core OpenEnv runtime (provides FastAPI server + HTTP client types)
+    # install from github
+    # "openenv-core[core] @ git+https://github.com/meta-pytorch/OpenEnv.git",
+    "openenv-core[core]>=0.2.1",
+    # Environment-specific dependencies
+    # Add all dependencies needed for your environment here
+    # Examples:
+    # "numpy>=1.19.0",
+    # "torch>=2.0.0",
+    # "gymnasium>=0.29.0",
+    # "openspiel>=1.0.0",
+    # "smolagents>=1.22.0,<2",
+]
+
+[project.optional-dependencies]
+dev = [
+    "pytest>=8.0.0",
+    "pytest-cov>=4.0.0",
+]
+
+[project.scripts]
+# Server entry point - enables running via: uv run --project . server
+# or: python -m ask_answer_env.server.app
+server = "ask_answer_env.server.app:main"
+
+[tool.setuptools]
+include-package-data = true
+packages = ["ask_answer_env", "ask_answer_env.server"]
+package-dir = { "ask_answer_env" = ".", "ask_answer_env.server" = "server" }

server/__init__.py

@@ -0,0 +1,11 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+"""Ask Answer Env environment server components."""
+
+from .ask_answer_env_environment import AskAnswerEnvironment
+
+__all__ = ["AskAnswerEnvironment"]

server/app.py

@@ -0,0 +1,81 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+"""
+FastAPI application for the Ask Answer Env Environment.
+
+This module creates an HTTP server that exposes the AskAnswerEnvironment
+over HTTP and WebSocket endpoints, compatible with EnvClient.
+
+Endpoints:
+    - POST /reset: Reset the environment
+    - POST /step: Execute an action
+    - GET /state: Get current environment state
+    - GET /schema: Get action/observation schemas
+    - WS /ws: WebSocket endpoint for persistent sessions
+
+Usage:
+    # Development (with auto-reload):
+    uvicorn server.app:app --reload --host 0.0.0.0 --port 8000
+
+    # Production:
+    uvicorn server.app:app --host 0.0.0.0 --port 8000 --workers 4
+
+    # Or run directly:
+    python -m server.app
+"""
+
+try:
+    from openenv.core.env_server.http_server import create_app
+except Exception as e:  # pragma: no cover
+    raise ImportError(
+        "openenv is required for the web interface. Install dependencies with '\n    uv sync\n'"
+    ) from e
+
+# Import from local models.py (PYTHONPATH includes /app/env in Docker)
+from models import AskAnswerAction, AskAnswerObservation
+from .ask_answer_env_environment import AskAnswerEnvironment
+
+
+# Create the app with web interface and README integration
+app = create_app(
+    AskAnswerEnvironment,
+    AskAnswerAction,
+    AskAnswerObservation,
+    env_name="ask_answer_env",
+    max_concurrent_envs=1,  # increase this number to allow more concurrent WebSocket sessions
+)
+
+
+def main(host: str = "0.0.0.0", port: int = 8000):
+    """
+    Entry point for direct execution via uv run or python -m.
+
+    This function enables running the server without Docker:
+        uv run --project . server
+        uv run --project . server --port 8001
+        python -m ask_answer_env.server.app
+
+    Args:
+        host: Host address to bind to (default: "0.0.0.0")
+        port: Port number to listen on (default: 8000)
+
+    For production deployments, consider using uvicorn directly with
+    multiple workers:
+        uvicorn ask_answer_env.server.app:app --workers 4
+    """
+    import uvicorn
+
+    uvicorn.run(app, host=host, port=port)
+
+
+if __name__ == "__main__":
+    import argparse
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--port", type=int, default=8000)
+    args = parser.parse_args()
+    main(port=args.port)

server/ask_answer_env_environment.py

@@ -0,0 +1,294 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+"""
+Ask Answer Env Environment Implementation.
+
+A deterministic slot-filling environment where agents must decide between
+asking clarifying questions or answering early to maximize reward.
+"""
+
+import random
+from typing import Optional
+from uuid import uuid4
+
+from openenv.core.env_server.interfaces import Environment
+from openenv.core.env_server.types import State
+
+from models import AskAnswerAction, AskAnswerObservation, KnownSlots
+
+
+# Constants
+CITIES = ["Paris", "Rome", "Tokyo", "Goa"]
+DATES = ["next_weekend", "mid_feb", "march"]
+BUDGETS = ["low", "mid", "high"]
+STYLES = ["relax", "adventure", "food"]  # Distractor slot
+MAX_STEPS = 3  # Forces agent to guess at least 1 core slot
+PROMPT = "Plan a short trip for me."
+
+# Rewards (unchanged from v0)
+STEP_PENALTY = -0.05
+ASK_UNKNOWN_REWARD = 0.1
+ASK_KNOWN_PENALTY = -0.2
+AUTO_FAIL_PENALTY = -1.0
+
+# Graded answer rewards (v1)
+ANSWER_CITY_CORRECT = 0.4
+ANSWER_DATE_CORRECT = 0.4
+ANSWER_BUDGET_CORRECT = 0.4
+ANSWER_STYLE_CORRECT_BONUS = 0.1  # Optional nice-to-have
+ANSWER_CORE_ALL_CORRECT_BONUS = 0.2
+ANSWER_CORE_ANY_WRONG_PENALTY = -0.6
+
+
+class AskAnswerEnvironment(Environment):
+    """
+    A slot-filling environment for training RL agents.
+
+    The agent must decide between:
+    - Asking clarifying questions (ASK) to reveal hidden slot values
+    - Answering early (ANSWER) to end the episode
+
+    Hidden state (city, date, budget, style) is sampled at reset with a seeded RNG.
+    The agent can ask about slots to reveal their values before answering.
+    
+    With MAX_STEPS=3, the agent can only ask 2 slots before being forced to answer,
+    creating a non-trivial ask-vs-act tradeoff. The "style" slot is a distractor
+    that provides less reward than core slots (city, date, budget).
+
+    Rewards:
+    - Step penalty: -0.05 per step
+    - ASK unknown slot: +0.1
+    - ASK known slot: -0.2
+    - ANSWER: graded per-slot (+0.4 each core, +0.1 style)
+    - Core all correct bonus: +0.2
+    - Core any wrong penalty: -0.6
+    - Auto-fail (steps exhausted): -1.0
+    """
+
+    SUPPORTS_CONCURRENT_SESSIONS: bool = True
+
+    def __init__(self):
+        """Initialize the ask_answer_env environment."""
+        self._state = State(episode_id=str(uuid4()), step_count=0)
+        self._rng: random.Random = random.Random()
+        
+        # Hidden truth (sampled at reset)
+        self._hidden_city: str = ""
+        self._hidden_date: str = ""
+        self._hidden_budget: str = ""
+        self._hidden_style: str = ""
+        
+        # Known slots (revealed through ASK actions)
+        self._known: KnownSlots = KnownSlots()
+        self._steps_left: int = MAX_STEPS
+        self._done: bool = False
+
+    def reset(self, seed: Optional[int] = None) -> AskAnswerObservation:
+        """
+        Reset the environment with optional seed for determinism.
+
+        Args:
+            seed: Random seed for reproducibility
+
+        Returns:
+            AskAnswerObservation with initial state
+        """
+        self._state = State(episode_id=str(uuid4()), step_count=0)
+        
+        # Initialize RNG with seed
+        if seed is not None:
+            self._rng = random.Random(seed)
+        else:
+            self._rng = random.Random()
+        
+        # Sample hidden truth
+        self._hidden_city = self._rng.choice(CITIES)
+        self._hidden_date = self._rng.choice(DATES)
+        self._hidden_budget = self._rng.choice(BUDGETS)
+        self._hidden_style = self._rng.choice(STYLES)
+        
+        # Reset known slots and step counter
+        self._known = KnownSlots()
+        self._steps_left = MAX_STEPS
+        self._done = False
+
+        return AskAnswerObservation(
+            prompt=PROMPT,
+            known=self._known,
+            steps_left=self._steps_left,
+            done=False,
+            reward=0.0,
+        )
+
+    def step(self, action: AskAnswerAction) -> AskAnswerObservation:  # type: ignore[override]
+        """
+        Execute a step in the environment.
+
+        Args:
+            action: AskAnswerAction with type 'ask' or 'answer'
+
+        Returns:
+            AskAnswerObservation with updated state and reward
+        """
+        if self._done:
+            return AskAnswerObservation(
+                prompt=PROMPT,
+                known=self._known,
+                steps_left=self._steps_left,
+                done=True,
+                reward=0.0,
+            )
+
+        self._state.step_count += 1
+        
+        # Always apply step penalty
+        reward = STEP_PENALTY
+        done = False
+
+        if action.type == "ask":
+            reward += self._handle_ask(action.slot)
+            self._steps_left -= 1
+            
+            # Check for auto-fail
+            if self._steps_left == 0:
+                reward = AUTO_FAIL_PENALTY
+                done = True
+                
+        elif action.type == "answer":
+            reward += self._handle_answer(action)
+            done = True
+
+        self._done = done
+
+        # Calculate core_correct_count when episode ends via ANSWER
+        core_correct_count = None
+        if done and action.type == "answer":
+            core_correct_count = sum([
+                action.city == self._hidden_city,
+                action.date == self._hidden_date,
+                action.budget == self._hidden_budget,
+            ])
+
+        return AskAnswerObservation(
+            prompt=PROMPT,
+            known=self._known,
+            steps_left=self._steps_left,
+            done=done,
+            reward=reward,
+            core_correct_count=core_correct_count,
+        )
+
+    def _handle_ask(self, slot: Optional[str]) -> float:
+        """
+        Handle ASK action - reveal a slot if unknown.
+
+        Args:
+            slot: The slot to ask about ('city', 'date', 'budget', or 'style')
+
+        Returns:
+            Reward for the ASK action
+        """
+        if slot == "city":
+            if self._known.city is not None:
+                return ASK_KNOWN_PENALTY
+            self._known = KnownSlots(
+                city=self._hidden_city,
+                date=self._known.date,
+                budget=self._known.budget,
+                style=self._known.style,
+            )
+            return ASK_UNKNOWN_REWARD
+            
+        elif slot == "date":
+            if self._known.date is not None:
+                return ASK_KNOWN_PENALTY
+            self._known = KnownSlots(
+                city=self._known.city,
+                date=self._hidden_date,
+                budget=self._known.budget,
+                style=self._known.style,
+            )
+            return ASK_UNKNOWN_REWARD
+            
+        elif slot == "budget":
+            if self._known.budget is not None:
+                return ASK_KNOWN_PENALTY
+            self._known = KnownSlots(
+                city=self._known.city,
+                date=self._known.date,
+                budget=self._hidden_budget,
+                style=self._known.style,
+            )
+            return ASK_UNKNOWN_REWARD
+            
+        elif slot == "style":
+            if self._known.style is not None:
+                return ASK_KNOWN_PENALTY
+            self._known = KnownSlots(
+                city=self._known.city,
+                date=self._known.date,
+                budget=self._known.budget,
+                style=self._hidden_style,
+            )
+            return ASK_UNKNOWN_REWARD
+        
+        # Invalid slot
+        return ASK_KNOWN_PENALTY
+
+    def _handle_answer(self, action: AskAnswerAction) -> float:
+        """
+        Handle ANSWER action with graded rewards.
+
+        Reward structure:
+        - Per-slot rewards: +0.4 for each correct core slot (city, date, budget)
+        - Style bonus: +0.1 if style provided and correct (ignored if None)
+        - Core bonus: +0.2 if all core slots correct
+        - Core penalty: -0.6 if any core slot wrong
+
+        Args:
+            action: The answer action with city, date, budget, style values
+
+        Returns:
+            Reward for the ANSWER action
+        """
+        reward = 0.0
+        
+        # Check core slots
+        city_correct = action.city == self._hidden_city
+        date_correct = action.date == self._hidden_date
+        budget_correct = action.budget == self._hidden_budget
+        
+        # Per-slot rewards for core slots
+        if city_correct:
+            reward += ANSWER_CITY_CORRECT
+        if date_correct:
+            reward += ANSWER_DATE_CORRECT
+        if budget_correct:
+            reward += ANSWER_BUDGET_CORRECT
+        
+        # Style bonus (only if provided and correct, ignored if None)
+        if action.style is not None and action.style == self._hidden_style:
+            reward += ANSWER_STYLE_CORRECT_BONUS
+        
+        # Core bonus/penalty
+        core_all_correct = city_correct and date_correct and budget_correct
+        if core_all_correct:
+            reward += ANSWER_CORE_ALL_CORRECT_BONUS
+        else:
+            reward += ANSWER_CORE_ANY_WRONG_PENALTY
+        
+        return reward
+
+    @property
+    def state(self) -> State:
+        """
+        Get the current environment state.
+
+        Returns:
+            Current State with episode_id and step_count
+        """
+        return self._state

server/requirements.txt

@@ -0,0 +1,6 @@
+openenv[core]>=0.2.0
+fastapi>=0.115.0
+uvicorn>=0.24.0
+
+
+