Spaces:

ShinnosukeU
/

agent_language

Sleeping

App Files Files Community

ShinnosukeU commited on Mar 8

Commit

734b153

verified ·

1 Parent(s): 1495b72

Upload folder using huggingface_hub

Browse files

Files changed (10) hide show

frontend/index.html +360 -0
models.py +6 -5
openenv_agent_language.egg-info/PKG-INFO +1 -0
openenv_agent_language.egg-info/SOURCES.txt +5 -1
openenv_agent_language.egg-info/requires.txt +1 -0
pyproject.toml +2 -1
server/agent_language_environment.py +18 -17
server/app.py +9 -0
server/evaluate_protocal.py +506 -0
uv.lock +2 -0

frontend/index.html ADDED Viewed

	@@ -0,0 +1,360 @@

+<!DOCTYPE html>
+<html lang="en">
+<head>
+  <meta charset="UTF-8" />
+  <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+  <title>Agent Language Environment</title>
+  <style>
+    * { box-sizing: border-box; margin: 0; padding: 0; }
+    body {
+      font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif;
+      background: #f5f5f7;
+      color: #1a1a1a;
+      min-height: 100vh;
+      padding: 24px;
+    }
+    .layout {
+      display: grid;
+      grid-template-columns: 1fr 1fr;
+      gap: 16px;
+      max-width: 1200px;
+      margin: 0 auto;
+    }
+    .left-col, .right-col {
+      display: flex;
+      flex-direction: column;
+      gap: 16px;
+    }
+    .card {
+      background: #fff;
+      border: 1px solid #e0e0e0;
+      border-radius: 8px;
+      padding: 20px;
+    }
+    .card h2 {
+      font-size: 1rem;
+      font-weight: 700;
+      margin-bottom: 12px;
+    }
+    label {
+      display: block;
+      font-size: 0.875rem;
+      margin-bottom: 6px;
+    }
+    label .required {
+      color: #e53e3e;
+    }
+    textarea {
+      width: 100%;
+      min-height: 90px;
+      padding: 10px 12px;
+      border: 2px solid #4a90d9;
+      border-radius: 6px;
+      font-size: 0.875rem;
+      font-family: inherit;
+      resize: vertical;
+      outline: none;
+      color: #555;
+    }
+    textarea:focus {
+      border-color: #2563eb;
+    }
+    .field-hint {
+      font-size: 0.75rem;
+      color: #888;
+      margin-top: 4px;
+      font-style: italic;
+    }
+    .btn {
+      display: inline-block;
+      padding: 8px 20px;
+      border: none;
+      border-radius: 6px;
+      font-size: 0.875rem;
+      font-weight: 500;
+      cursor: pointer;
+      transition: opacity 0.15s;
+    }
+    .btn:hover { opacity: 0.85; }
+    .btn:active { opacity: 0.7; }
+    .btn:disabled { opacity: 0.5; cursor: not-allowed; }
+    .btn-primary { background: #4a7fd4; color: #fff; }
+    .btn-secondary { background: #5a6472; color: #fff; }
+    .btn-row {
+      display: flex;
+      gap: 10px;
+    }
+    .state-card .state-row {
+      font-size: 0.875rem;
+      color: #555;
+      margin-bottom: 6px;
+    }
+    .state-card .state-row strong {
+      color: #1a1a1a;
+    }
+    .obs-pre {
+      background: #f7f8fa;
+      border-radius: 4px;
+      padding: 12px;
+      font-family: "SFMono-Regular", Consolas, "Liberation Mono", Menlo, monospace;
+      font-size: 0.8rem;
+      white-space: pre;
+      overflow-x: auto;
+      line-height: 1.6;
+    }
+    .history-empty {
+      font-size: 0.875rem;
+      color: #555;
+    }
+    .history-list {
+      display: flex;
+      flex-direction: column;
+      gap: 10px;
+      max-height: 400px;
+      overflow-y: auto;
+    }
+    .history-item {
+      background: #f7f8fa;
+      border-radius: 6px;
+      padding: 10px 12px;
+      font-size: 0.8rem;
+    }
+    .history-item .hi-step {
+      font-weight: 600;
+      margin-bottom: 4px;
+      color: #333;
+    }
+    .history-item .hi-action {
+      color: #555;
+      margin-bottom: 2px;
+    }
+    .history-item .hi-reward {
+      color: #2563eb;
+      font-size: 0.75rem;
+    }
+    .error-msg {
+      color: #c53030;
+      font-size: 0.8rem;
+      margin-top: 8px;
+    }
+  </style>
+</head>
+<body>
+  <div class="layout">
+    <!-- LEFT COLUMN -->
+    <div class="left-col">
+      <!-- Take Action Card -->
+      <div class="card">
+        <h2>Take Action</h2>
+        <label>Language Specification <span class="required">*</span>:</label>
+        <textarea id="messageInput" placeholder="Enter language specification..."></textarea>
+        <div class="field-hint">Language Specification</div>
+        <div id="stepError" class="error-msg" style="display:none;"></div>
+        <div style="margin-top:12px;">
+          <button class="btn btn-primary" id="stepBtn" onclick="doStep()">Step</button>
+        </div>
+      </div>
+      <!-- Env Controls -->
+      <div class="btn-row">
+        <button class="btn btn-secondary" id="resetBtn" onclick="doReset()">Reset Environment</button>
+        <button class="btn btn-secondary" id="stateBtn" onclick="doGetState()">Get State</button>
+      </div>
+      <!-- Current State Card -->
+      <div class="card state-card">
+        <h2>Current State</h2>
+        <div class="state-row">Status: <strong id="stateStatus">—</strong></div>
+        <div class="state-row">Episode ID: <strong id="stateEpisodeId">—</strong></div>
+        <div class="state-row">Step Count: <strong id="stateStepCount">—</strong></div>
+        <div id="stateError" class="error-msg" style="display:none;"></div>
+      </div>
+    </div>
+    <!-- RIGHT COLUMN -->
+    <div class="right-col">
+      <!-- Current Observation Card -->
+      <div class="card">
+        <h2>Current Observation</h2>
+        <div class="obs-pre" id="obsDisplay">—</div>
+      </div>
+      <!-- Action History Card -->
+      <div class="card">
+        <h2>Action History</h2>
+        <div id="historyContainer">
+          <div class="history-empty" id="historyEmpty">No actions taken yet</div>
+          <div class="history-list" id="historyList" style="display:none;"></div>
+        </div>
+      </div>
+    </div>
+  </div>
+  <script>
+    const BASE_URL = 'http://localhost:8000';
+    const actionHistory = [];
+    function setButtons(disabled) {
+      ['stepBtn', 'resetBtn', 'stateBtn'].forEach(id => {
+        document.getElementById(id).disabled = disabled;
+      });
+    }
+    function showError(elemId, msg) {
+      const el = document.getElementById(elemId);
+      el.textContent = msg;
+      el.style.display = 'block';
+    }
+    function clearError(elemId) {
+      const el = document.getElementById(elemId);
+      el.textContent = '';
+      el.style.display = 'none';
+    }
+    function updateStateDisplay(data) {
+      document.getElementById('stateStatus').textContent = data.status ?? '—';
+      document.getElementById('stateEpisodeId').textContent = data.episode_id ?? '—';
+      document.getElementById('stateStepCount').textContent = data.step_count ?? '—';
+    }
+    function updateObsDisplay(obs) {
+      // Show obs as pretty JSON, excluding internal fields if desired
+      const display = {};
+      for (const [k, v] of Object.entries(obs)) {
+        if (!['done', 'reward', 'metadata'].includes(k)) {
+          display[k] = v;
+        }
+      }
+      document.getElementById('obsDisplay').textContent = JSON.stringify(display, null, 2);
+    }
+    function addToHistory(stepNum, message, reward) {
+      actionHistory.push({ stepNum, message, reward });
+      renderHistory();
+    }
+    function renderHistory() {
+      const empty = document.getElementById('historyEmpty');
+      const list = document.getElementById('historyList');
+      if (actionHistory.length === 0) {
+        empty.style.display = '';
+        list.style.display = 'none';
+        return;
+      }
+      empty.style.display = 'none';
+      list.style.display = 'flex';
+      list.innerHTML = actionHistory.slice().reverse().map(item => `
+        <div class="history-item">
+          <div class="hi-step">Step ${item.stepNum}</div>
+          <div class="hi-action">Spec: "${item.message}"</div>
+          <div class="hi-reward">Reward: ${item.reward.toFixed(2)}</div>
+        </div>
+      `).join('');
+    }
+    async function doReset() {
+      clearError('stateError');
+      setButtons(true);
+      try {
+        const res = await fetch(`${BASE_URL}/reset`, { method: 'POST' });
+        if (!res.ok) throw new Error(`HTTP ${res.status}`);
+        const data = await res.json();
+        // data contains observation + state
+        if (data.observation) updateObsDisplay(data.observation);
+        if (data.state) updateStateDisplay({ ...data.state, status: 'Reset' });
+        // Clear history on reset
+        actionHistory.length = 0;
+        renderHistory();
+      } catch (e) {
+        showError('stateError', `Reset failed: ${e.message}`);
+      } finally {
+        setButtons(false);
+      }
+    }
+    async function doGetState() {
+      clearError('stateError');
+      setButtons(true);
+      try {
+        const res = await fetch(`${BASE_URL}/state`);
+        if (!res.ok) throw new Error(`HTTP ${res.status}`);
+        const data = await res.json();
+        updateStateDisplay(data);
+      } catch (e) {
+        showError('stateError', `Get state failed: ${e.message}`);
+      } finally {
+        setButtons(false);
+      }
+    }
+    async function doStep() {
+      clearError('stepError');
+      const message = document.getElementById('messageInput').value.trim();
+      if (!message) {
+        showError('stepError', 'Language Specification is required.');
+        return;
+      }
+      setButtons(true);
+      try {
+        const res = await fetch(`${BASE_URL}/step`, {
+          method: 'POST',
+          headers: { 'Content-Type': 'application/json' },
+          body: JSON.stringify({ language_specification: message }),
+        });
+        if (!res.ok) throw new Error(`HTTP ${res.status}`);
+        const data = await res.json();
+        const obs = data.observation ?? data;
+        updateObsDisplay(obs);
+        if (data.state) {
+          updateStateDisplay({ ...data.state, status: 'Running' });
+          addToHistory(data.state.step_count, message, obs.reward ?? 0);
+        } else {
+          addToHistory(actionHistory.length + 1, message, obs.reward ?? 0);
+        }
+        document.getElementById('messageInput').value = '';
+      } catch (e) {
+        showError('stepError', `Step failed: ${e.message}`);
+      } finally {
+        setButtons(false);
+      }
+    }
+    // Load initial state on page load
+    (async () => {
+      try {
+        const res = await fetch(`${BASE_URL}/state`);
+        if (res.ok) {
+          const data = await res.json();
+          updateStateDisplay(data);
+        }
+      } catch (_) {}
+    })();
+  </script>
+</body>
+</html>

models.py CHANGED Viewed

@@ -10,19 +10,20 @@ Data models for the Agent Language Environment.
 The agent_language environment is a simple test environment that echoes back messages.
 """
 from pydantic import Field
-from openenv.core.env_server.types import Action, Observation
 class AgentLanguageAction(Action):
     """Action for the Agent Language environment - just a message to echo."""
-    message: str = Field(..., description="Message to echo back")
 class AgentLanguageObservation(Observation):
     """Observation from the Agent Language environment - the echoed message."""
-    echoed_message: str = Field(default="", description="The echoed message")
-    message_length: int = Field(default=0, description="Length of the echoed message")

 The agent_language environment is a simple test environment that echoes back messages.
 """
+from openenv.core.env_server.types import Action, Observation, State
 from pydantic import Field
 class AgentLanguageAction(Action):
     """Action for the Agent Language environment - just a message to echo."""
+    language_specification: str = Field(..., description="Language Specification")
 class AgentLanguageObservation(Observation):
     """Observation from the Agent Language environment - the echoed message."""
+    message: str = Field(default="", description="Scenario")
+class AgentLanguageState(State):
+    """Custom state fields."""

openenv_agent_language.egg-info/PKG-INFO CHANGED Viewed

@@ -3,6 +3,7 @@ Name: openenv-agent_language
 Version: 0.1.0
 Summary: Agent Language environment for OpenEnv
 Requires-Python: >=3.10
 Requires-Dist: openenv-core[core]>=0.2.0
 Provides-Extra: dev
 Requires-Dist: pytest>=8.0.0; extra == "dev"

 Version: 0.1.0
 Summary: Agent Language environment for OpenEnv
 Requires-Python: >=3.10
+Requires-Dist: openai>=2.26.0
 Requires-Dist: openenv-core[core]>=0.2.0
 Provides-Extra: dev
 Requires-Dist: pytest>=8.0.0; extra == "dev"

openenv_agent_language.egg-info/SOURCES.txt CHANGED Viewed

@@ -1,4 +1,7 @@
 README.md
 pyproject.toml
 ./__init__.py
 ./client.py
@@ -11,4 +14,5 @@ openenv_agent_language.egg-info/requires.txt
 openenv_agent_language.egg-info/top_level.txt
 server/__init__.py
 server/agent_language_environment.py
-server/app.py

 README.md
+__init__.py
+client.py
+models.py
 pyproject.toml
 ./__init__.py
 ./client.py
 openenv_agent_language.egg-info/top_level.txt
 server/__init__.py
 server/agent_language_environment.py
+server/app.py
+server/evaluate_protocal.py

openenv_agent_language.egg-info/requires.txt CHANGED Viewed

@@ -1,3 +1,4 @@
 openenv-core[core]>=0.2.0
 [dev]

+openai>=2.26.0
 openenv-core[core]>=0.2.0
 [dev]

pyproject.toml CHANGED Viewed

@@ -14,6 +14,7 @@ version = "0.1.0"
 description = "Agent Language environment for OpenEnv"
 requires-python = ">=3.10"
 dependencies = [
     # Core OpenEnv runtime (provides FastAPI server + HTTP client types)
     # install from github
     # "openenv-core[core] @ git+https://github.com/meta-pytorch/OpenEnv.git",
@@ -42,4 +43,4 @@ server = "agent_language.server.app:main"
 [tool.setuptools]
 include-package-data = true
 packages = ["agent_language", "agent_language.server"]
-package-dir = { "agent_language" = ".", "agent_language.server" = "server" }

 description = "Agent Language environment for OpenEnv"
 requires-python = ">=3.10"
 dependencies = [
+    "openai>=2.26.0",
     # Core OpenEnv runtime (provides FastAPI server + HTTP client types)
     # install from github
     # "openenv-core[core] @ git+https://github.com/meta-pytorch/OpenEnv.git",
 [tool.setuptools]
 include-package-data = true
 packages = ["agent_language", "agent_language.server"]
+package-dir = { "agent_language" = ".", "agent_language.server" = "server" }

server/agent_language_environment.py CHANGED Viewed

@@ -13,11 +13,15 @@ Perfect for testing HTTP server infrastructure.
 from uuid import uuid4
 from openenv.core.env_server.interfaces import Environment
-from openenv.core.env_server.types import State
-from models import AgentLanguageAction, AgentLanguageObservation
 class AgentLanguageEnvironment(Environment):
     """
@@ -42,23 +46,26 @@ class AgentLanguageEnvironment(Environment):
     # getting their own environment instance (when using factory mode in app.py).
     SUPPORTS_CONCURRENT_SESSIONS: bool = True
-    def __init__(self):
         """Initialize the agent_language environment."""
-        self._state = State(episode_id=str(uuid4()), step_count=0)
         self._reset_count = 0
-    def reset(self) -> AgentLanguageObservation:
         """
         Reset the environment.
         Returns:
             AgentLanguageObservation with a ready message
         """
-        self._state = State(episode_id=str(uuid4()), step_count=0)
         self._reset_count += 1
         return AgentLanguageObservation(
-            echoed_message="Agent Language environment ready!",
             message_length=0,
             done=False,
             reward=0.0,
@@ -75,23 +82,17 @@ class AgentLanguageEnvironment(Environment):
             AgentLanguageObservation with the echoed message and its length
         """
         self._state.step_count += 1
         message = action.message
-        length = len(message)
-        # Simple reward: longer messages get higher rewards
-        reward = length * 0.1
         return AgentLanguageObservation(
             echoed_message=message,
-            message_length=length,
-            done=False,
             reward=reward,
-            metadata={"original_message": message, "step": self._state.step_count},
         )
     @property
-    def state(self) -> State:
         """
         Get the current environment state.

 from uuid import uuid4
+from evaluate_protocal import evaluate_lang_spec
+from models import AgentLanguageAction, AgentLanguageObservation, AgentLanguageState
 from openenv.core.env_server.interfaces import Environment
+COMMUNICATION_PROTOCAL_PROMPT = """"You are generating a communication protocol between two agents. Produce language specification that **minimize** the number of tokens needed for a single exchange while preserving clarity. This could be some abbreviation synonyms or some template for the communication. Your communication protocal might be detailed and should include examples of the communication. Try not to limit the amount of actual information that is passed to each agent. Instead forcus on formtting of the communication, and telling the agents to abbreviate and make the communication as short as possible. The communication protocal itsefl does not need to be concise, it should be in natural language with full sentences, even paragraphs if needed, and easy to understand.
+Example:
+When you communicate, avoid extra greetings.
+"""
 class AgentLanguageEnvironment(Environment):
     """
     # getting their own environment instance (when using factory mode in app.py).
     SUPPORTS_CONCURRENT_SESSIONS: bool = True
+    def __init__(self, seed):
         """Initialize the agent_language environment."""
+        self._state = AgentLanguageState(episode_id=str(uuid4()), step_count=0)
         self._reset_count = 0
+        self.seed = seed
+    def reset(self, seed) -> AgentLanguageObservation:
         """
         Reset the environment.
         Returns:
             AgentLanguageObservation with a ready message
         """
+        self._state = AgentLanguageState(episode_id=str(uuid4()), step_count=0)
         self._reset_count += 1
+        message = COMMUNICATION_PROTOCAL_PROMPT + "\n\n Design a communication protocal for two agents scheduling a meeting time."
         return AgentLanguageObservation(
+            message=message,
             message_length=0,
             done=False,
             reward=0.0,
             AgentLanguageObservation with the echoed message and its length
         """
         self._state.step_count += 1
         message = action.message
+        reward = evaluate_lang_spec(message)
         return AgentLanguageObservation(
             echoed_message=message,
+            done=True,
             reward=reward,
+            #metadata={"original_message": message, "step": self._state.step_count},
         )
     @property
+    def state(self) -> AgentLanguageState:
         """
         Get the current environment state.

server/app.py CHANGED Viewed

@@ -35,6 +35,8 @@ except Exception as e:  # pragma: no cover
         "openenv is required for the web interface. Install dependencies with '\n    uv sync\n'"
     ) from e
 # Import from local models.py (PYTHONPATH includes /app/env in Docker)
 from models import AgentLanguageAction, AgentLanguageObservation
 from .agent_language_environment import AgentLanguageEnvironment
@@ -49,6 +51,13 @@ app = create_app(
     max_concurrent_envs=1,  # increase this number to allow more concurrent WebSocket sessions
 )
 def main(host: str = "0.0.0.0", port: int = 8000):
     """

         "openenv is required for the web interface. Install dependencies with '\n    uv sync\n'"
     ) from e
+from fastapi.middleware.cors import CORSMiddleware
 # Import from local models.py (PYTHONPATH includes /app/env in Docker)
 from models import AgentLanguageAction, AgentLanguageObservation
 from .agent_language_environment import AgentLanguageEnvironment
     max_concurrent_envs=1,  # increase this number to allow more concurrent WebSocket sessions
 )
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
 def main(host: str = "0.0.0.0", port: int = 8000):
     """

server/evaluate_protocal.py ADDED Viewed

	@@ -0,0 +1,506 @@

+import json
+import os
+import random
+import re
+import sys
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+from pathlib import Path
+from dotenv import load_dotenv
+from openai import OpenAI
+from openai.types.chat import (
+    ChatCompletionAssistantMessageParam,
+    ChatCompletionMessageParam,
+    ChatCompletionSystemMessageParam,
+    ChatCompletionUserMessageParam,
+)
+load_dotenv()
+TASK_COMPLETE_KEYWORD = "TASK_COMPLETE"
+MAX_TURNS = 30
+RESULTS_FILE = "results.json"
+@dataclass
+class TimeSlot:
+    day: str
+    location: str
+    start: float  # hours in 24h (e.g. 10.5 = 10:30)
+    end: float
+    def contains(self, day: str, location: str, time: float, duration: float) -> bool:
+        return (
+            self.day == day
+            and self.location.lower() == location.lower()
+            and self.start <= time
+            and time + duration <= self.end
+        )
+@dataclass
+class Schedule:
+    name: str
+    slots: list[TimeSlot]
+    def is_available(
+        self, day: str, location: str, time: float, duration: float
+    ) -> bool:
+        return any(slot.contains(day, location, time, duration) for slot in self.slots)
+    def to_natural(self) -> str:
+        day_names = {
+            "Mo": "Monday",
+            "Tu": "Tuesday",
+            "We": "Wednesday",
+            "Th": "Thursday",
+            "Fr": "Friday",
+        }
+        parts = []
+        for slot in self.slots:
+            start_str = _format_time(slot.start)
+            end_str = _format_time(slot.end)
+            parts.append(
+                f"{day_names[slot.day]} in {slot.location}, {start_str}-{end_str}"
+            )
+        return "; ".join(parts)
+def _format_time(t: float) -> str:
+    hours = int(t)
+    minutes = int((t - hours) * 60)
+    if minutes == 0:
+        return str(hours)
+    return f"{hours}:{minutes:02d}"
+def _parse_time(s: str) -> float:
+    if ":" in s:
+        h, m = s.split(":")
+        return int(h) + int(m) / 60
+    return float(s)
+def verify_meeting(
+    schedules: list[Schedule], day: str, location: str, time: float, duration: float
+) -> tuple[bool, list[str]]:
+    errors = []
+    for schedule in schedules:
+        if not schedule.is_available(day, location, time, duration):
+            time_str = _format_time(time)
+            errors.append(
+                f"{schedule.name} is NOT available on {day} at {time_str} ({location})"
+            )
+    return len(errors) == 0, errors
+DAY_ALIASES: dict[str, str] = {
+    "monday": "Mo",
+    "tuesday": "Tu",
+    "wednesday": "We",
+    "thursday": "Th",
+    "friday": "Fr",
+    "mon": "Mo",
+    "tue": "Tu",
+    "wed": "We",
+    "thu": "Th",
+    "fri": "Fr",
+    "mo": "Mo",
+    "tu": "Tu",
+    "we": "We",
+    "th": "Th",
+    "fr": "Fr",
+}
+def parse_compact_result(text: str) -> tuple[str, str, float] | None:
+    pattern = r"=>\s*([A-Za-z]{2,9})\[([A-Za-z]+)\](\d{1,2}(?::\d{2})?)\s*-\s*\d{1,2}(?::\d{2})?"
+    match = re.search(pattern, text)
+    if not match:
+        return None
+    raw_day = match.group(1).lower()
+    day = DAY_ALIASES.get(raw_day, match.group(1))
+    location = match.group(2)
+    time = _parse_time(match.group(3))
+    return day, location, time
+@dataclass
+class Session:
+    client: OpenAI
+    model: str
+    name: str = ""
+    system_prompt: str = ""
+    messages: list[ChatCompletionMessageParam] = field(default_factory=list)
+    total_completion_tokens: int = 0
+    turns: int = 0
+    def __post_init__(self) -> None:
+        if self.system_prompt:
+            sys_msg: ChatCompletionSystemMessageParam = {
+                "role": "system",
+                "content": self.system_prompt,
+            }
+            self.messages.append(sys_msg)
+    def send(self, content: str) -> str:
+        user_msg: ChatCompletionUserMessageParam = {
+            "role": "user",
+            "content": content,
+        }
+        self.messages.append(user_msg)
+        response = self.client.chat.completions.create(
+            model=self.model,
+            messages=self.messages,
+            max_tokens=500,
+        )
+        assistant_content = response.choices[0].message.content or ""
+        assistant_msg: ChatCompletionAssistantMessageParam = {
+            "role": "assistant",
+            "content": assistant_content,
+        }
+        self.messages.append(assistant_msg)
+        if response.usage:
+            self.total_completion_tokens += response.usage.completion_tokens
+        self.turns += 1
+        return assistant_content
+    def is_complete(self) -> bool:
+        if not self.messages:
+            return False
+        last = self.messages[-1]
+        content = last.get("content")
+        return (
+            last["role"] == "assistant"
+            and isinstance(content, str)
+            and (TASK_COMPLETE_KEYWORD in content or "=>" in content)
+        )
+def negotiate(
+    agent_a: Session, agent_b: Session, max_turns: int = MAX_TURNS
+) -> list[dict[str, str]]:
+    conversation: list[dict[str, str]] = []
+    response = agent_a.send("Propose a meeting time.")
+    conversation.append({"agent": agent_a.name, "content": response})
+    for _ in range(max_turns):
+        if agent_a.is_complete():
+            break
+        response = agent_b.send(response)
+        conversation.append({"agent": agent_b.name, "content": response})
+        if agent_b.is_complete():
+            break
+        response = agent_a.send(response)
+        conversation.append({"agent": agent_a.name, "content": response})
+    return conversation
+MEETING_DURATION = 30  # minutes
+DAYS = ["Mo", "Tu", "We", "Th", "Fr"]
+CITIES = ["SF", "NYC"]
+MIN_HOUR = 8
+MAX_HOUR = 18
+def generate_schedules(
+    num_overlaps: int, rng: random.Random
+) -> tuple[Schedule, Schedule]:
+    days = DAYS[:]
+    rng.shuffle(days)
+    overlap_days = days[:num_overlaps]
+    filler_days = days[num_overlaps:]
+    a_slots: list[TimeSlot] = []
+    b_slots: list[TimeSlot] = []
+    for day in overlap_days:
+        city = rng.choice(CITIES)
+        overlap_start = rng.randint(MIN_HOUR + 1, MAX_HOUR - 2)
+        overlap_end = rng.randint(
+            overlap_start + 1, min(overlap_start + 3, MAX_HOUR - 1)
+        )
+        a_start = rng.randint(MIN_HOUR, overlap_start)
+        a_end = rng.randint(overlap_end, MAX_HOUR)
+        b_start = rng.randint(MIN_HOUR, overlap_start)
+        b_end = rng.randint(overlap_end, MAX_HOUR)
+        a_slots.append(TimeSlot(day, city, float(a_start), float(a_end)))
+        b_slots.append(TimeSlot(day, city, float(b_start), float(b_end)))
+    for day in filler_days:
+        strategy = rng.choice(["a_only", "b_only", "diff_cities"])
+        if strategy == "a_only":
+            city = rng.choice(CITIES)
+            start = rng.randint(MIN_HOUR, MAX_HOUR - 2)
+            end = rng.randint(start + 2, MAX_HOUR)
+            a_slots.append(TimeSlot(day, city, float(start), float(end)))
+        elif strategy == "b_only":
+            city = rng.choice(CITIES)
+            start = rng.randint(MIN_HOUR, MAX_HOUR - 2)
+            end = rng.randint(start + 2, MAX_HOUR)
+            b_slots.append(TimeSlot(day, city, float(start), float(end)))
+        else:
+            city_a, city_b = rng.sample(CITIES, 2)
+            start_a = rng.randint(MIN_HOUR, MAX_HOUR - 2)
+            end_a = rng.randint(start_a + 2, MAX_HOUR)
+            start_b = rng.randint(MIN_HOUR, MAX_HOUR - 2)
+            end_b = rng.randint(start_b + 2, MAX_HOUR)
+            a_slots.append(TimeSlot(day, city_a, float(start_a), float(end_a)))
+            b_slots.append(TimeSlot(day, city_b, float(start_b), float(end_b)))
+    day_order = {d: i for i, d in enumerate(DAYS)}
+    a_slots.sort(key=lambda s: day_order[s.day])
+    b_slots.sort(key=lambda s: day_order[s.day])
+    return Schedule("T", a_slots), Schedule("J", b_slots)
+def compute_valid_meetings(
+    sched_a: Schedule, sched_b: Schedule, duration: float
+) -> list[dict[str, str | float]]:
+    valid: list[dict[str, str | float]] = []
+    for slot_a in sched_a.slots:
+        for slot_b in sched_b.slots:
+            if (
+                slot_a.day != slot_b.day
+                or slot_a.location.lower() != slot_b.location.lower()
+            ):
+                continue
+            overlap_start = max(slot_a.start, slot_b.start)
+            overlap_end = min(slot_a.end, slot_b.end)
+            if overlap_end - overlap_start >= duration:
+                valid.append(
+                    {
+                        "day": slot_a.day,
+                        "location": slot_a.location,
+                        "start": overlap_start,
+                        "end": overlap_end,
+                    }
+                )
+    return valid
+def run_trial(
+    client: OpenAI,
+    model: str,
+    lang_spec: str,
+    rng: random.Random,
+) -> dict:
+    num_overlaps = rng.choice([0, 1, 2])
+    t_schedule, j_schedule = generate_schedules(num_overlaps, rng)
+    duration = MEETING_DURATION / 60
+    valid_meetings = compute_valid_meetings(t_schedule, j_schedule, duration)
+    agent_t = Session(
+        client=client,
+        model=model,
+        name="T",
+        system_prompt=(
+            f"You are T. Your availability: {t_schedule.to_natural()}\n"
+            f"Meeting duration: {MEETING_DURATION} minutes.\n" + RULES + lang_spec
+        ),
+    )
+    agent_j = Session(
+        client=client,
+        model=model,
+        name="J",
+        system_prompt=(
+            f"You are J. Your availability: {j_schedule.to_natural()}\n"
+            f"Meeting duration: {MEETING_DURATION} minutes.\n" + RULES + lang_spec
+        ),
+    )
+    conversation = negotiate(agent_t, agent_j)
+    combined_completion_tokens = (
+        agent_t.total_completion_tokens + agent_j.total_completion_tokens
+    )
+    # Check if agents said NO_VALID_TIME
+    said_no_valid = any("NO_VALID_TIME" in msg["content"] for msg in conversation)
+    # Check if agents proposed a meeting
+    meeting_result = None
+    for msg in reversed(conversation):
+        parsed = parse_compact_result(msg["content"])
+        if parsed:
+            meeting_result = parsed
+            break
+    correct = False
+    errors: list[str] = []
+    if said_no_valid and not meeting_result:
+        if not valid_meetings:
+            correct = True
+        else:
+            errors.append("Agent said NO_VALID_TIME but valid meetings exist")
+    elif meeting_result:
+        day, location, time = meeting_result
+        correct, errors = verify_meeting(
+            [t_schedule, j_schedule], day, location, time, duration
+        )
+    else:
+        errors.append("No meeting proposed and no NO_VALID_TIME signal")
+    combined_chars = sum(len(msg["content"]) for msg in conversation)
+    return {
+        "correct": correct,
+        "errors": errors,
+        "num_overlaps": num_overlaps,
+        "valid_meetings": valid_meetings,
+        "schedules": {
+            "T": t_schedule.to_natural(),
+            "J": j_schedule.to_natural(),
+        },
+        "combined_completion_tokens": combined_completion_tokens,
+        "combined_chars": combined_chars,
+        "total_turns": agent_t.turns + agent_j.turns,
+        "agents": {
+            agent_t.name: {
+                "turns": agent_t.turns,
+                "completion_tokens": agent_t.total_completion_tokens,
+            },
+            agent_j.name: {
+                "turns": agent_j.turns,
+                "completion_tokens": agent_j.total_completion_tokens,
+            },
+        },
+        "meeting": (
+            {
+                "day": meeting_result[0],
+                "location": meeting_result[1],
+                "time": meeting_result[2],
+            }
+            if meeting_result
+            else None
+        ),
+        "conversation": conversation,
+    }
+def run_experiment(
+    client: OpenAI,
+    model: str,
+    lang_spec: str,
+    n: int,
+    experiment_id: str | None = None,
+) -> dict:
+    exp_id = experiment_id or "unnamed"
+    rng = random.Random()
+    trials = []
+    for i in range(n):
+        trial = run_trial(client, model, lang_spec, rng)
+        trials.append(trial)
+        status = "CORRECT" if trial["correct"] else "INCORRECT"
+        print(
+            f"[{i + 1}/{n}] {status} | "
+            f"chars={trial['combined_chars']} | "
+            f"tokens={trial['combined_completion_tokens']} | "
+            f"turns={trial['total_turns']}"
+        )
+    experiment = {
+        "experiment_id": exp_id,
+        "model": model,
+        "lang_spec": lang_spec,
+        "num_trials": n,
+        "created_at": datetime.now(timezone.utc).isoformat(),
+        "trials": trials,
+    }
+    path = Path(RESULTS_FILE)
+    results: list[dict] = []
+    if path.exists():
+        results = json.loads(path.read_text())
+    results.append(experiment)
+    path.write_text(json.dumps(results, indent=2) + "\n")
+    correct_count = sum(1 for t in trials if t["correct"])
+    chars = [t["combined_chars"] for t in trials]
+    tokens = [t["combined_completion_tokens"] for t in trials]
+    print(
+        f"\nExperiment {exp_id}: "
+        f"{correct_count}/{n} correct | "
+        f"mean_chars={sum(chars) / len(chars):.0f} | "
+        f"mean_tokens={sum(tokens) / len(tokens):.0f}"
+    )
+    return experiment
+RULES = """\
+Rules:
+- You can ONLY be in the city listed for each day. You CANNOT travel or change cities.
+- You can ONLY meet if BOTH people are in the SAME city on the SAME day.
+- Reject any proposal where you are in a different city than the other person.
+- When agreed, respond with => <day>[<city>]<start>-<end> and TASK_COMPLETE (e.g. => Fr[NYC]9-9:30)
+- If no valid meeting time exists, respond with NO_VALID_TIME and TASK_COMPLETE
+"""
+LANG_SPECS: dict[str, str] = {
+    "compact": """\
+You communicate using a compact scheduling protocol. Here is the format:
+M? d=<minutes> z=<timezone> w=<day range> p=<preference>
+<name>: <day>[<city>]<start>-<end>,<start>-<end>;<day>[<city>]<start>-<end>
+=> <day>[<city>]<start>-<end>
+Example:
+M? d=30 z=ET w=Mo-Fr p=earliest
+T: Mo[SF]9-12;Tu[NYC]13-17;Th[SF]10-15;Fr[NYC]9-11
+J: Mo[NYC]10-14;Tu[SF]9-12;We[SF]13-16;Th[NYC]11-15;Fr[NYC]9-11
+=> Fr[NYC]9-9:30
+- Times are in 24h format
+- Days: Mo,Tu,We,Th,Fr
+- Locations in brackets: [SF], [NYC]
+- You MUST use this compact format for ALL messages, no natural language
+- To propose: send your available slots in compact format
+- To accept: respond with => <day>[<city>]<start>-<end>
+- To reject/counter: send your slots that conflict and suggest alternatives
+""",
+    "natural": """\
+Negotiate with the other person to find a 30-minute in-person meeting time.
+Keep responses short (1-2 sentences).
+""",
+}
+def main() -> None:
+    client = OpenAI(
+        base_url="https://openrouter.ai/api/v1",
+        api_key=os.environ["OPENROUTER_API_KEY"],
+    )
+    model = "google/gemini-3-flash-preview"
+    n = int(sys.argv[1]) if len(sys.argv) > 1 else 1
+    for spec_name, lang_spec in LANG_SPECS.items():
+        run_experiment(client, model, lang_spec, n, spec_name)
+def evaluate_lang_spec(lang_spec: str, n: int = 5) -> float:
+    client = OpenAI(
+        base_url="https://openrouter.ai/api/v1",
+        api_key=os.environ["OPENROUTER_API_KEY"],
+    )
+    model = "google/gemini-3-flash-preview"
+    rng = random.Random()
+    trials = [run_trial(client, model, lang_spec, rng) for _ in range(n)]
+    return sum(t["combined_completion_tokens"] for t in trials) / len(trials)
+if __name__ == "__main__":
+    main()

uv.lock CHANGED Viewed

@@ -1075,6 +1075,7 @@ name = "openenv-agent-language"
 version = "0.1.0"
 source = { editable = "." }
 dependencies = [
     { name = "openenv-core", extra = ["core"] },
 ]
@@ -1086,6 +1087,7 @@ dev = [
 [package.metadata]
 requires-dist = [
     { name = "openenv-core", extras = ["core"], specifier = ">=0.2.0" },
     { name = "pytest", marker = "extra == 'dev'", specifier = ">=8.0.0" },
     { name = "pytest-cov", marker = "extra == 'dev'", specifier = ">=4.0.0" },

 version = "0.1.0"
 source = { editable = "." }
 dependencies = [
+    { name = "openai" },
     { name = "openenv-core", extra = ["core"] },
 ]
 [package.metadata]
 requires-dist = [
+    { name = "openai", specifier = ">=2.26.0" },
     { name = "openenv-core", extras = ["core"], specifier = ">=0.2.0" },
     { name = "pytest", marker = "extra == 'dev'", specifier = ">=8.0.0" },
     { name = "pytest-cov", marker = "extra == 'dev'", specifier = ">=4.0.0" },