Upload folder using huggingface_hub

evaluate_protocal.py

@@ -3,6 +3,7 @@ import os
 import random
 import re
 import sys
+from concurrent.futures import ThreadPoolExecutor, as_completed
 from dataclasses import dataclass, field
 from datetime import datetime, timezone
 from pathlib import Path
@@ -395,21 +396,29 @@ def run_experiment(
     lang_spec: str,
     n: int,
     experiment_id: str | None = None,
+    max_workers: int = 8,
 ) -> dict:
     exp_id = experiment_id or "unnamed"
-    rng = random.Random()
-    trials = []
-
-    for i in range(n):
-        trial = run_trial(client, model, lang_spec, rng)
-        trials.append(trial)
-        status = "CORRECT" if trial["correct"] else "INCORRECT"
-        print(
-            f"[{i + 1}/{n}] {status} | "
-            f"chars={trial['combined_chars']} | "
-            f"tokens={trial['combined_completion_tokens']} | "
-            f"turns={trial['total_turns']}"
-        )
+    trials = [None] * n
+
+    def _run(i: int) -> tuple[int, dict]:
+        rng = random.Random()
+        return i, run_trial(client, model, lang_spec, rng)
+
+    completed = 0
+    with ThreadPoolExecutor(max_workers=max_workers) as executor:
+        futures = {executor.submit(_run, i): i for i in range(n)}
+        for future in as_completed(futures):
+            i, trial = future.result()
+            trials[i] = trial
+            completed += 1
+            status = "CORRECT" if trial["correct"] else "INCORRECT"
+            print(
+                f"[{completed}/{n}] {status} | "
+                f"chars={trial['combined_chars']} | "
+                f"tokens={trial['combined_completion_tokens']} | "
+                f"turns={trial['total_turns']}"
+            )
 
     experiment = {
         "experiment_id": exp_id,
@@ -497,8 +506,15 @@ def evaluate_lang_spec(lang_spec: str, n: int = 5) -> float:
         api_key=os.environ["OPENROUTER_API_KEY"],
     )
     model = "google/gemini-3-flash-preview"
-    rng = random.Random()
-    trials = [run_trial(client, model, lang_spec, rng) for _ in range(n)]
+
+    def _run(_: int) -> dict:
+        return run_trial(client, model, lang_spec, random.Random())
+
+    with ThreadPoolExecutor(max_workers=n) as executor:
+        trials = list(executor.map(_run, range(n)))
+
+    Path("sample_conversation.json").write_text(json.dumps(trials[0], indent=2) + "\n")
+
     return sum(t["combined_completion_tokens"] for t in trials) / len(trials)
 
 

frontend/index.html

@@ -325,7 +325,7 @@
         const res = await fetch(`${BASE_URL}/step`, {
           method: 'POST',
           headers: { 'Content-Type': 'application/json' },
-          body: JSON.stringify({ language_specification: message }),
+          body: JSON.stringify({ action: { language_specification: message } }),
         });
         if (!res.ok) throw new Error(`HTTP ${res.status}`);
         const data = await res.json();
@@ -333,9 +333,9 @@
         updateObsDisplay(obs);
         if (data.state) {
           updateStateDisplay({ ...data.state, status: 'Running' });
-          addToHistory(data.state.step_count, message, obs.reward ?? 0);
+          addToHistory(data.state.step_count, message, data.reward ?? 0);
         } else {
-          addToHistory(actionHistory.length + 1, message, obs.reward ?? 0);
+          addToHistory(actionHistory.length + 1, message, data.reward ?? 0);
         }
         document.getElementById('messageInput').value = '';
       } catch (e) {

openenv_agent_language.egg-info/PKG-INFO

@@ -5,6 +5,7 @@ Summary: Agent Language environment for OpenEnv
 Requires-Python: >=3.10
 Requires-Dist: openai>=2.26.0
 Requires-Dist: openenv-core[core]>=0.2.0
+Requires-Dist: python-dotenv>=1.2.2
 Provides-Extra: dev
 Requires-Dist: pytest>=8.0.0; extra == "dev"
 Requires-Dist: pytest-cov>=4.0.0; extra == "dev"

openenv_agent_language.egg-info/SOURCES.txt

@@ -5,6 +5,7 @@ models.py
 pyproject.toml
 ./__init__.py
 ./client.py
+./evaluate_protocal.py
 ./models.py
 openenv_agent_language.egg-info/PKG-INFO
 openenv_agent_language.egg-info/SOURCES.txt
@@ -14,5 +15,4 @@ openenv_agent_language.egg-info/requires.txt
 openenv_agent_language.egg-info/top_level.txt
 server/__init__.py
 server/agent_language_environment.py
-server/app.py
-server/evaluate_protocal.py
+server/app.py

openenv_agent_language.egg-info/requires.txt

@@ -1,5 +1,6 @@
 openai>=2.26.0
 openenv-core[core]>=0.2.0
+python-dotenv>=1.2.2
 
 [dev]
 pytest>=8.0.0

pyproject.toml

@@ -27,6 +27,7 @@ dependencies = [
     # "gymnasium>=0.29.0",
     # "openspiel>=1.0.0",
     # "smolagents>=1.22.0,<2",
+    "python-dotenv>=1.2.2",
 ]
 
 [project.optional-dependencies]

sample_conversation.json

@@ -0,0 +1,39 @@
+{
+  "correct": false,
+  "errors": [
+    "Agent said NO_VALID_TIME but valid meetings exist"
+  ],
+  "num_overlaps": 1,
+  "valid_meetings": [
+    {
+      "day": "Th",
+      "location": "NYC",
+      "start": 11.0,
+      "end": 15.0
+    }
+  ],
+  "schedules": {
+    "T": "Monday in SF, 10-16; Thursday in NYC, 11-15",
+    "J": "Monday in NYC, 16-18; Tuesday in SF, 10-16; Wednesday in SF, 12-17; Thursday in NYC, 10-18; Friday in SF, 16-18"
+  },
+  "combined_completion_tokens": 34,
+  "combined_chars": 132,
+  "total_turns": 1,
+  "agents": {
+    "T": {
+      "turns": 1,
+      "completion_tokens": 34
+    },
+    "J": {
+      "turns": 0,
+      "completion_tokens": 0
+    }
+  },
+  "meeting": null,
+  "conversation": [
+    {
+      "agent": "T",
+      "content": "Please provide the details (M?, T:, J:) for the meeting request so I can evaluate the slots. Otherwise:\n\nNO_VALID_TIME\nTASK_COMPLETE"
+    }
+  ]
+}

server/.env

@@ -0,0 +1 @@
+OPENROUTER_API_KEY=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

server/agent_language_environment.py

@@ -46,13 +46,12 @@ class AgentLanguageEnvironment(Environment):
     # getting their own environment instance (when using factory mode in app.py).
     SUPPORTS_CONCURRENT_SESSIONS: bool = True
 
-    def __init__(self, seed):
+    def __init__(self):
         """Initialize the agent_language environment."""
         self._state = AgentLanguageState(episode_id=str(uuid4()), step_count=0)
         self._reset_count = 0
-        self.seed = seed
 
-    def reset(self, seed) -> AgentLanguageObservation:
+    def reset(self) -> AgentLanguageObservation:
         """
         Reset the environment.
 
@@ -83,7 +82,7 @@ class AgentLanguageEnvironment(Environment):
         """
         self._state.step_count += 1
         language_specification = action.language_specification
-        reward = evaluate_lang_spec(language_specification)
+        reward = -evaluate_lang_spec(language_specification)
         return AgentLanguageObservation(
             message="Do not call any more function.",
             done=True,

server/app.py

@@ -35,12 +35,17 @@ except Exception as e:  # pragma: no cover
         "openenv is required for the web interface. Install dependencies with '\n    uv sync\n'"
     ) from e
 
+import os
+
+import dotenv
 from fastapi.middleware.cors import CORSMiddleware
 
+dotenv.load_dotenv()
+
 # Import from local models.py (PYTHONPATH includes /app/env in Docker)
 from models import AgentLanguageAction, AgentLanguageObservation
-from .agent_language_environment import AgentLanguageEnvironment
 
+from .agent_language_environment import AgentLanguageEnvironment
 
 # Create the app with web interface and README integration
 app = create_app(

uv.lock

@@ -1077,6 +1077,7 @@ source = { editable = "." }
 dependencies = [
     { name = "openai" },
     { name = "openenv-core", extra = ["core"] },
+    { name = "python-dotenv" },
 ]
 
 [package.optional-dependencies]
@@ -1091,6 +1092,7 @@ requires-dist = [
     { name = "openenv-core", extras = ["core"], specifier = ">=0.2.0" },
     { name = "pytest", marker = "extra == 'dev'", specifier = ">=8.0.0" },
     { name = "pytest-cov", marker = "extra == 'dev'", specifier = ">=4.0.0" },
+    { name = "python-dotenv", specifier = ">=1.2.2" },
 ]
 provides-extras = ["dev"]