Initial Freeciv OpenEnv Space

.dockerignore

@@ -0,0 +1,14 @@
+.git
+.venv
+__pycache__/
+*.pyc
+.pytest_cache/
+build/
+pres/
+freeciv_env.egg-info/
+notes.md
+outline.md
+hackathon.md
+qwen35_live_long_trainer_state.json
+freeciv_rl_training_curve.png
+tests/

.gitignore

@@ -0,0 +1 @@
+**/__pycache__

AGENTS.md

@@ -0,0 +1,5 @@
+This is a hackathon project--see the brief at hackathon.md. Always read the file before responding.
+Please keep your answers concise.
+
+We have an H100. Access it by running this command: `northflank ssh service --projectId hackathon --serviceId civ --proxyOnly` (may already be running)
+then `ssh root@127.0.0.1 -p 35731` (or whatever the `northflank ssh` command prints out as the command to run)

Dockerfile

@@ -0,0 +1,31 @@
+FROM omkarasoftware/freeciv-web:latest
+
+USER root
+RUN apt-get update && apt-get install -y \
+    git \
+    curl \
+    ca-certificates \
+ && rm -rf /var/lib/apt/lists/*
+RUN mkdir -p /app/env && chown -R docker:docker /app
+
+USER docker
+ENV HOME=/home/docker
+WORKDIR /app/env
+
+COPY --chown=docker:docker . /app/env
+RUN chmod +x /app/env/scripts/start_space.sh
+RUN curl -LsSf https://astral.sh/uv/install.sh | sh
+ENV PATH="/app/env/.venv/bin:/home/docker/.local/bin:$PATH"
+RUN uv python install 3.11
+RUN uv venv --python 3.11 /app/env/.venv
+RUN UV_PROJECT_ENVIRONMENT=/app/env/.venv uv sync --frozen --no-dev --no-editable
+
+ENV PYTHONPATH="/app/env:$PYTHONPATH"
+ENV ENABLE_WEB_INTERFACE=true
+ENV FREECIV_SERVER_URL=http://127.0.0.1
+ENV FREECIV_TURN_TIMEOUT_S=120
+
+HEALTHCHECK --interval=30s --timeout=5s --start-period=20s --retries=10 \
+    CMD curl -f http://localhost:8000/health || exit 1
+
+CMD ["/app/env/scripts/start_space.sh"]

README.md

@@ -1,10 +1,117 @@
 ---
-title: Openenv Hack
-emoji: 😻
-colorFrom: red
-colorTo: yellow
+title: Freeciv Environment Server
+emoji: 🎮
+colorFrom: blue
+colorTo: indigo
 sdk: docker
 pinned: false
+app_port: 8000
+base_path: /web
+tags:
+  - openenv
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# freeciv-env
+
+OpenEnv environment for Freeciv, built on top of `freeciv-bot`.
+
+## Current scope
+
+This environment exposes a small, trainable action surface:
+
+- `end_turn`
+- `move_unit(unit_id, direction)`
+- `build_city(unit_id)`
+- `set_city_production(city_id, target)`
+- `set_research(tech_name)`
+
+Observations are text-first and include compact structured summaries of:
+
+- current turn
+- score
+- known and visible map tiles
+- units
+- cities
+- legal actions
+
+## Local development
+
+Install dependencies:
+
+```bash
+uv sync --extra dev
+```
+
+Run tests:
+
+```bash
+uv run pytest
+```
+
+Run the server:
+
+```bash
+uv run uvicorn freeciv_env.server.app:app --host 0.0.0.0 --port 8000
+```
+
+Run the fast GRPO loop:
+
+```bash
+uv sync --extra dev --extra train
+uv run python scripts/train_grpo_fast.py --env-url http://127.0.0.1 --max-steps 50
+```
+
+## Hackathon / Unsloth notes
+
+For the hackathon Colab submission path on H100s, Unsloth recommended the BF16 OpenEnv gpt-oss 20B notebook:
+
+- <https://colab.research.google.com/github/unslothai/notebooks/blob/main/nb/OpenEnv_gpt_oss_(20B)_Reinforcement_Learning_2048_Game_BF16.ipynb>
+
+If you adapt that notebook for this environment, reduce `max_steps` to `300` for a faster run.
+
+Useful notebook indexes:
+
+- RL notebooks: <https://unsloth.ai/docs/get-started/unsloth-notebooks#grpo-reasoning-rl>
+- all notebooks: <https://unsloth.ai/docs/get-started/unsloth-notebooks>
+- notebook repo: <https://github.com/unslothai/notebooks/tree/main/nb>
+
+If GRPO is too slow, start from a smaller notebook with `fast_inference = True` and add the Freeciv/OpenEnv calls:
+
+- <https://colab.research.google.com/github/unslothai/notebooks/blob/main/nb/Qwen3_(4B)-GRPO.ipynb>
+- <https://colab.research.google.com/github/unslothai/notebooks/blob/main/nb/Advanced_Llama3_2_(3B)_GRPO_LoRA.ipynb>
+
+If vLLM GRPO fails, Unsloth suggested a clean virtualenv install:
+
+```bash
+python -m venv unsloth_env
+source unsloth_env/bin/activate
+pip install --upgrade pip && pip install uv
+uv pip install unsloth vllm --torch-backend=auto
+```
+
+If Unsloth is already installed, update it for the latest GRPO fixes:
+
+```bash
+pip install --upgrade --no-cache-dir --no-deps unsloth unsloth_zoo
+```
+
+## Live runtime requirements
+
+The default server app uses `freeciv-bot` against a local Freeciv Web runtime.
+
+Environment variables:
+
+- `FREECIV_SERVER_URL` (default: `http://127.0.0.1`)
+- `FREECIV_USERNAME` (default: `openenvbot`)
+- `FREECIV_CLIENT_PORT` (default: `6000`)
+- `FREECIV_TURN_TIMEOUT_S` (default: `60`)
+
+The included automated tests use a fake session backend, so they do not require a live Freeciv server.
+
+The GRPO training script uses:
+
+- `Qwen/Qwen3.5-0.8B`
+- Unsloth bf16 LoRA loading
+- TRL `GRPOTrainer`
+- integer-only action selection to minimize generated tokens
+- offline GRPO over env-sampled states for maximum throughput

__init__.py

@@ -0,0 +1 @@
+from freeciv_env import *

build/lib/freeciv_env/__init__.py

@@ -0,0 +1,10 @@
+from freeciv_env.client import FreecivEnv
+from freeciv_env.models import FreecivAction, FreecivObservation, FreecivState, LegalAction
+
+__all__ = [
+    "FreecivAction",
+    "FreecivEnv",
+    "FreecivObservation",
+    "FreecivState",
+    "LegalAction",
+]

build/lib/freeciv_env/adapter.py

@@ -0,0 +1,335 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any
+
+from freeciv_env.models import CitySummary, FreecivAction, FreecivObservation, LegalAction, UnitSummary
+
+
+ActionLookupKey = tuple[str, int | None, int | None, str | None]
+
+
+@dataclass(frozen=True)
+class ActionRef:
+    controller: str
+    actor_id: int | str
+    raw_action_key: str
+
+
+@dataclass
+class RawSnapshot:
+    turn: int
+    state: dict[str, Any]
+    actions: dict[str, Any]
+
+
+@dataclass(frozen=True)
+class SnapshotMetrics:
+    score: float
+    known_tiles: int
+    visible_tiles: int
+    city_count: int
+    unit_count: int
+    techs_researched: int
+
+
+@dataclass
+class PreparedObservation:
+    observation: FreecivObservation
+    metrics: SnapshotMetrics
+    action_refs: dict[ActionLookupKey, ActionRef]
+
+
+def _map_status_rows(raw_state: dict[str, Any]) -> list[list[int | float]]:
+    raw_map = raw_state.get("map", {})
+    status = raw_map.get("status", [])
+    return status if isinstance(status, list) else []
+
+
+def count_known_tiles(raw_state: dict[str, Any]) -> int:
+    return sum(1 for row in _map_status_rows(raw_state) for value in row if value and value > 0)
+
+
+def count_visible_tiles(raw_state: dict[str, Any]) -> int:
+    return sum(1 for row in _map_status_rows(raw_state) for value in row if value and value >= 2)
+
+
+def extract_metrics(snapshot: RawSnapshot) -> SnapshotMetrics:
+    player = snapshot.state.get("player", {})
+    return SnapshotMetrics(
+        score=float(player.get("my_score", 0.0)),
+        known_tiles=count_known_tiles(snapshot.state),
+        visible_tiles=count_visible_tiles(snapshot.state),
+        city_count=len(snapshot.state.get("city", {})),
+        unit_count=len(snapshot.state.get("unit", {})),
+        techs_researched=int(player.get("my_techs_researched", 0) or 0),
+    )
+
+
+def action_lookup_key(action: FreecivAction) -> ActionLookupKey:
+    if action.action_type == "move_unit":
+        return ("move_unit", action.unit_id, action.direction, None)
+    if action.action_type == "build_city":
+        return ("build_city", action.unit_id, None, None)
+    if action.action_type == "set_city_production":
+        return ("set_city_production", action.city_id, None, action.target)
+    if action.action_type == "set_research":
+        return ("set_research", None, None, action.target)
+    return ("end_turn", None, None, None)
+
+
+def _parse_target_name(raw_action_key: str, prefix: str) -> str:
+    suffix = raw_action_key.removeprefix(prefix)
+    name, _sep, _tail = suffix.rpartition("_")
+    return name or suffix
+
+
+
+def _controller_actions(snapshot: RawSnapshot, controller: str) -> dict[str, Any]:
+    raw_actions = snapshot.actions.get(controller, {})
+    if isinstance(raw_actions, dict):
+        return raw_actions
+    if hasattr(raw_actions, "json_struct"):
+        json_actions = raw_actions.json_struct()
+        return json_actions if isinstance(json_actions, dict) else {}
+    return {}
+
+
+
+def _extract_legal_actions(snapshot: RawSnapshot) -> tuple[list[LegalAction], dict[ActionLookupKey, ActionRef]]:
+    legal_actions: list[LegalAction] = [
+        LegalAction(
+            action_type="end_turn",
+            label="End the current turn",
+            raw_action_key="__end_turn__",
+        )
+    ]
+    refs: dict[ActionLookupKey, ActionRef] = {}
+
+    for actor_id, action_map in _controller_actions(snapshot, "unit").items():
+        unit_id = int(actor_id)
+        if action_map.get("build"):
+            legal_actions.append(
+                LegalAction(
+                    action_type="build_city",
+                    label=f"Build a city with unit {unit_id}",
+                    unit_id=unit_id,
+                    raw_action_key="build",
+                )
+            )
+            refs[("build_city", unit_id, None, None)] = ActionRef(
+                controller="unit",
+                actor_id=unit_id,
+                raw_action_key="build",
+            )
+        for raw_action_key, enabled in sorted(action_map.items()):
+            if not enabled or not raw_action_key.startswith("goto_"):
+                continue
+            direction = int(raw_action_key.split("_", 1)[1])
+            legal_actions.append(
+                LegalAction(
+                    action_type="move_unit",
+                    label=f"Move unit {unit_id} in direction {direction}",
+                    unit_id=unit_id,
+                    direction=direction,
+                    raw_action_key=raw_action_key,
+                )
+            )
+            refs[("move_unit", unit_id, direction, None)] = ActionRef(
+                controller="unit",
+                actor_id=unit_id,
+                raw_action_key=raw_action_key,
+            )
+
+    for actor_id, action_map in _controller_actions(snapshot, "city").items():
+        city_id = int(actor_id)
+        for raw_action_key, enabled in sorted(action_map.items()):
+            if not enabled:
+                continue
+            if raw_action_key.startswith("change_unit_prod_"):
+                target = _parse_target_name(raw_action_key, "change_unit_prod_")
+            elif raw_action_key.startswith("change_improve_prod_"):
+                target = _parse_target_name(raw_action_key, "change_improve_prod_")
+            else:
+                continue
+            legal_actions.append(
+                LegalAction(
+                    action_type="set_city_production",
+                    label=f"Set city {city_id} production to {target}",
+                    city_id=city_id,
+                    target=target,
+                    raw_action_key=raw_action_key,
+                )
+            )
+            refs[("set_city_production", city_id, None, target)] = ActionRef(
+                controller="city",
+                actor_id=city_id,
+                raw_action_key=raw_action_key,
+            )
+
+    tech_actions = _controller_actions(snapshot, "tech").get("cur_player", {})
+    for raw_action_key, enabled in sorted(tech_actions.items()):
+        if not enabled or not raw_action_key.startswith("research_tech_"):
+            continue
+        target = _parse_target_name(raw_action_key, "research_tech_")
+        legal_actions.append(
+            LegalAction(
+                action_type="set_research",
+                label=f"Research {target}",
+                target=target,
+                raw_action_key=raw_action_key,
+            )
+        )
+        refs[("set_research", None, None, target)] = ActionRef(
+            controller="tech",
+            actor_id="cur_player",
+            raw_action_key=raw_action_key,
+        )
+
+    legal_actions.sort(
+        key=lambda item: (
+            item.action_type,
+            item.unit_id or -1,
+            item.city_id or -1,
+            item.direction or -1,
+            item.target or "",
+        )
+    )
+    return legal_actions, refs
+
+
+def _extract_unit_summaries(snapshot: RawSnapshot) -> list[UnitSummary]:
+    unit_actions = _controller_actions(snapshot, "unit")
+    units: list[UnitSummary] = []
+    for actor_id, unit in sorted(snapshot.state.get("unit", {}).items(), key=lambda item: int(item[0])):
+        action_map = unit_actions.get(str(actor_id), unit_actions.get(actor_id, {}))
+        move_directions = sorted(
+            int(raw_action_key.split("_", 1)[1])
+            for raw_action_key, enabled in action_map.items()
+            if enabled and raw_action_key.startswith("goto_")
+        )
+        units.append(
+            UnitSummary(
+                unit_id=int(actor_id),
+                unit_type=str(unit.get("type_rule_name", "Unknown")),
+                health=int(unit.get("health", 0) or 0),
+                moves_left=int(unit.get("moves_left", unit.get("movesleft", 0)) or 0),
+                home_city_id=(
+                    int(unit.get("home_city"))
+                    if unit.get("home_city") not in (None, -1, "")
+                    else None
+                ),
+                veteran_level=int(unit.get("veteran", 0) or 0),
+                can_build_city=bool(action_map.get("build", False)),
+                move_directions=move_directions,
+            )
+        )
+    return units
+
+
+def _extract_city_summaries(snapshot: RawSnapshot) -> list[CitySummary]:
+    city_actions = _controller_actions(snapshot, "city")
+    cities: list[CitySummary] = []
+    for actor_id, city in sorted(snapshot.state.get("city", {}).items(), key=lambda item: int(item[0])):
+        action_map = city_actions.get(str(actor_id), city_actions.get(actor_id, {}))
+        production_options = [
+            _parse_target_name(raw_action_key, "change_unit_prod_")
+            for raw_action_key, enabled in sorted(action_map.items())
+            if enabled and raw_action_key.startswith("change_unit_prod_")
+        ] + [
+            _parse_target_name(raw_action_key, "change_improve_prod_")
+            for raw_action_key, enabled in sorted(action_map.items())
+            if enabled and raw_action_key.startswith("change_improve_prod_")
+        ]
+        cities.append(
+            CitySummary(
+                city_id=int(actor_id),
+                size=int(city.get("size", 0) or 0),
+                prod_food=int(city.get("prod_food", 0) or 0),
+                prod_shield=int(city.get("prod_shield", 0) or 0),
+                prod_trade=int(city.get("prod_trade", 0) or 0),
+                surplus_food=int(city.get("surplus_food", 0) or 0),
+                surplus_shield=int(city.get("surplus_shield", 0) or 0),
+                surplus_trade=int(city.get("surplus_trade", 0) or 0),
+                production_kind=(
+                    int(city.get("production_kind"))
+                    if city.get("production_kind") is not None
+                    else None
+                ),
+                production_value=(
+                    int(city.get("production_value"))
+                    if city.get("production_value") is not None
+                    else None
+                ),
+                turns_to_complete=(
+                    float(city.get("turns_to_prod_complete"))
+                    if city.get("turns_to_prod_complete") is not None
+                    else None
+                ),
+                production_options=production_options,
+            )
+        )
+    return cities
+
+
+def _build_summary(
+    snapshot: RawSnapshot,
+    metrics: SnapshotMetrics,
+    units: list[UnitSummary],
+    cities: list[CitySummary],
+    legal_actions: list[LegalAction],
+) -> str:
+    player = snapshot.state.get("player", {})
+    lines = [
+        f"Turn {snapshot.turn}",
+        f"Score {metrics.score:.1f}",
+        f"Map: {metrics.known_tiles} known tiles, {metrics.visible_tiles} visible tiles",
+        f"Economy: {player.get('my_gold', 0)} gold, science rate {player.get('my_science', 0)}%",
+        f"Cities: {metrics.city_count}",
+    ]
+    for city in cities[:5]:
+        lines.append(
+            f"- City {city.city_id}: size {city.size}, food {city.prod_food}/{city.surplus_food:+d}, "
+            f"shields {city.prod_shield}/{city.surplus_shield:+d}, trade {city.prod_trade}/{city.surplus_trade:+d}"
+        )
+    lines.append(f"Units: {metrics.unit_count}")
+    for unit in units[:8]:
+        lines.append(
+            f"- Unit {unit.unit_id}: {unit.unit_type}, hp {unit.health}, moves_left {unit.moves_left}, "
+            f"build_city={str(unit.can_build_city).lower()}, move_dirs={unit.move_directions}"
+        )
+    lines.append(f"Techs researched: {metrics.techs_researched}")
+    lines.append(f"Legal actions exposed: {len(legal_actions)}")
+    return "\n".join(lines)
+
+
+def prepare_observation(
+    snapshot: RawSnapshot,
+    *,
+    reward: float,
+    done: bool,
+    status: str,
+    metadata: dict[str, Any] | None = None,
+) -> PreparedObservation:
+    legal_actions, action_refs = _extract_legal_actions(snapshot)
+    metrics = extract_metrics(snapshot)
+    units = _extract_unit_summaries(snapshot)
+    cities = _extract_city_summaries(snapshot)
+    observation = FreecivObservation(
+        turn=snapshot.turn,
+        score=metrics.score,
+        known_tiles=metrics.known_tiles,
+        visible_tiles=metrics.visible_tiles,
+        city_count=metrics.city_count,
+        unit_count=metrics.unit_count,
+        techs_researched=metrics.techs_researched,
+        status=status,
+        summary=_build_summary(snapshot, metrics, units, cities, legal_actions),
+        units=units,
+        cities=cities,
+        legal_actions=legal_actions,
+        reward=reward,
+        done=done,
+        metadata=metadata or {},
+    )
+    return PreparedObservation(observation=observation, metrics=metrics, action_refs=action_refs)

build/lib/freeciv_env/client.py

@@ -0,0 +1,22 @@
+from __future__ import annotations
+
+from openenv.core.client_types import StepResult
+from openenv.core.env_client import EnvClient
+
+from freeciv_env.models import FreecivAction, FreecivObservation, FreecivState
+
+
+class FreecivEnv(EnvClient[FreecivAction, FreecivObservation, FreecivState]):
+    def _step_payload(self, action: FreecivAction) -> dict:
+        return action.model_dump(exclude_none=True)
+
+    def _parse_result(self, payload: dict) -> StepResult[FreecivObservation]:
+        observation = FreecivObservation(**payload["observation"])
+        return StepResult(
+            observation=observation,
+            reward=payload.get("reward"),
+            done=payload.get("done", False),
+        )
+
+    def _parse_state(self, payload: dict) -> FreecivState:
+        return FreecivState(**payload)

build/lib/freeciv_env/grpo.py

@@ -0,0 +1,97 @@
+from __future__ import annotations
+
+import re
+from typing import Iterable
+
+from freeciv_env.models import FreecivAction, FreecivObservation, LegalAction
+
+SYSTEM_PROMPT = (
+    "You are choosing the next action for a Freeciv agent. "
+    "Return only the integer index of the best legal action. "
+    "Do not output words, punctuation, JSON, or explanations."
+)
+
+TASK_PROMPT = (
+    "Pick the legal action index that maximizes immediate reward. "
+    "Invalid actions are penalized. Shorter outputs are better."
+)
+
+
+def format_action_line(index: int, action: LegalAction) -> str:
+    return f"{index}: {action.label}"
+
+
+def build_turn_prompt(observation: FreecivObservation, task_prompt: str = TASK_PROMPT) -> str:
+    action_lines = [format_action_line(index, action) for index, action in enumerate(observation.legal_actions)]
+    return (
+        f"{task_prompt}\n\n"
+        f"State:\n{observation.summary}\n\n"
+        f"Legal actions:\n" + "\n".join(action_lines) + "\n\n"
+        "Return exactly one integer index."
+    )
+
+
+def parse_action_choice(completion_text: str, legal_actions: Iterable[LegalAction]) -> FreecivAction | None:
+    legal_actions = list(legal_actions)
+    match = re.search(r"-?\d+", completion_text)
+    if match is None:
+        return None
+    index = int(match.group(0))
+    if index < 0 or index >= len(legal_actions):
+        return None
+    action = legal_actions[index]
+    if action.action_type == "end_turn":
+        return FreecivAction(action_type="end_turn")
+    if action.action_type == "move_unit":
+        return FreecivAction(action_type="move_unit", unit_id=action.unit_id, direction=action.direction)
+    if action.action_type == "build_city":
+        return FreecivAction(action_type="build_city", unit_id=action.unit_id)
+    if action.action_type == "set_city_production":
+        return FreecivAction(action_type="set_city_production", city_id=action.city_id, target=action.target)
+    if action.action_type == "set_research":
+        return FreecivAction(action_type="set_research", target=action.target)
+    raise ValueError(f"unsupported action_type: {action.action_type}")
+
+
+def action_priority(action: LegalAction) -> tuple[int, int]:
+    if action.action_type == "build_city":
+        return (500, 0)
+    if action.action_type == "set_research":
+        return (400, 0)
+    if action.action_type == "set_city_production":
+        bonus = 50 if (action.target or "") == "Settlers" else 0
+        return (300 + bonus, 0)
+    if action.action_type == "move_unit":
+        return (200, -(action.direction or 0))
+    if action.action_type == "end_turn":
+        return (0, 0)
+    return (-1000, 0)
+
+
+
+def oracle_action_index(legal_actions: Iterable[LegalAction]) -> int:
+    legal_actions = list(legal_actions)
+    if not legal_actions:
+        raise ValueError("no legal actions available")
+    best_index = 0
+    best_priority = action_priority(legal_actions[0])
+    for index, action in enumerate(legal_actions[1:], start=1):
+        priority = action_priority(action)
+        if priority > best_priority:
+            best_index = index
+            best_priority = priority
+    return best_index
+
+
+
+def reward_from_oracle(completions, best_index, **kwargs):
+    del kwargs
+    rewards = []
+    for completion, expected in zip(completions, best_index):
+        match = re.search(r"-?\d+", completion if isinstance(completion, str) else str(completion))
+        if match is None:
+            rewards.append(-0.25)
+            continue
+        chosen = int(match.group(0))
+        rewards.append(1.0 if chosen == int(expected) else 0.0)
+    return rewards

build/lib/freeciv_env/models.py

@@ -0,0 +1,112 @@
+from __future__ import annotations
+
+from typing import Literal
+
+from pydantic import BaseModel, Field, model_validator
+
+from openenv.core.env_server.types import Action, Observation, State
+
+
+class UnitSummary(BaseModel):
+    unit_id: int = Field(..., description="Freeciv unit id")
+    unit_type: str = Field(..., description="Ruleset unit type name")
+    health: int = Field(0, description="Current health")
+    moves_left: int = Field(0, description="Movement points remaining")
+    home_city_id: int | None = Field(None, description="Home city id, if any")
+    veteran_level: int = Field(0, description="Veteran level")
+    can_build_city: bool = Field(False, description="Whether the unit can found a city now")
+    move_directions: list[int] = Field(default_factory=list, description="Legal move direction indexes")
+
+
+class CitySummary(BaseModel):
+    city_id: int = Field(..., description="Freeciv city id")
+    size: int = Field(..., description="Population size")
+    prod_food: int = Field(0, description="Gross food output")
+    prod_shield: int = Field(0, description="Gross shield output")
+    prod_trade: int = Field(0, description="Gross trade output")
+    surplus_food: int = Field(0, description="Net food surplus")
+    surplus_shield: int = Field(0, description="Net shield surplus")
+    surplus_trade: int = Field(0, description="Net trade surplus")
+    production_kind: int | None = Field(None, description="Current production kind enum from Freeciv")
+    production_value: int | None = Field(None, description="Current production value id from Freeciv")
+    turns_to_complete: float | None = Field(None, description="Turns until current production completes")
+    production_options: list[str] = Field(default_factory=list, description="Legal production targets")
+
+
+class LegalAction(BaseModel):
+    action_type: Literal[
+        "end_turn",
+        "move_unit",
+        "build_city",
+        "set_city_production",
+        "set_research",
+    ]
+    label: str = Field(..., description="Human-readable action label")
+    unit_id: int | None = Field(None, description="Target unit id")
+    city_id: int | None = Field(None, description="Target city id")
+    direction: int | None = Field(None, description="Freeciv direction index 0..7")
+    target: str | None = Field(None, description="Production or tech target name")
+    raw_action_key: str | None = Field(None, description="Underlying freeciv-bot action key")
+
+
+class FreecivAction(Action):
+    action_type: Literal[
+        "end_turn",
+        "move_unit",
+        "build_city",
+        "set_city_production",
+        "set_research",
+    ]
+    unit_id: int | None = None
+    city_id: int | None = None
+    direction: int | None = None
+    target: str | None = None
+
+    @model_validator(mode="after")
+    def validate_shape(self) -> "FreecivAction":
+        if self.action_type == "end_turn":
+            return self
+        if self.action_type == "move_unit":
+            if self.unit_id is None or self.direction is None:
+                raise ValueError("move_unit requires unit_id and direction")
+            return self
+        if self.action_type == "build_city":
+            if self.unit_id is None:
+                raise ValueError("build_city requires unit_id")
+            return self
+        if self.action_type == "set_city_production":
+            if self.city_id is None or not self.target:
+                raise ValueError("set_city_production requires city_id and target")
+            return self
+        if self.action_type == "set_research":
+            if not self.target:
+                raise ValueError("set_research requires target")
+            return self
+        raise ValueError(f"unsupported action_type: {self.action_type}")
+
+
+class FreecivObservation(Observation):
+    turn: int = Field(..., description="Current game turn")
+    score: float = Field(..., description="Current player score")
+    known_tiles: int = Field(..., description="Tiles known to the player")
+    visible_tiles: int = Field(..., description="Tiles currently visible to the player")
+    city_count: int = Field(..., description="Number of owned cities")
+    unit_count: int = Field(..., description="Number of owned units")
+    techs_researched: int = Field(..., description="Number of researched techs")
+    status: str = Field("ok", description="High-level environment status")
+    summary: str = Field(..., description="Compact text summary for LLMs")
+    units: list[UnitSummary] = Field(default_factory=list, description="Compact unit summaries")
+    cities: list[CitySummary] = Field(default_factory=list, description="Compact city summaries")
+    legal_actions: list[LegalAction] = Field(default_factory=list, description="Legal actions exposed by the environment")
+    reward: float = Field(0.0, description="Reward from the last action")
+    done: bool = Field(False, description="Whether the episode is done")
+
+
+class FreecivState(State):
+    turn: int = Field(0, description="Current game turn")
+    score: float = Field(0.0, description="Current player score")
+    known_tiles: int = Field(0, description="Known tiles")
+    visible_tiles: int = Field(0, description="Visible tiles")
+    city_count: int = Field(0, description="Owned city count")
+    unit_count: int = Field(0, description="Owned unit count")
+    techs_researched: int = Field(0, description="Researched tech count")

build/lib/freeciv_env/runtime.py

@@ -0,0 +1,401 @@
+from __future__ import annotations
+
+import asyncio
+import json
+import threading
+import time
+from typing import Protocol
+from urllib.parse import urlencode, urlparse
+from urllib.request import Request, urlopen
+
+from freeciv_env.adapter import ActionRef, RawSnapshot
+
+
+class FreecivSession(Protocol):
+    def reset(self, seed: int | None = None) -> RawSnapshot: ...
+
+    def apply_action(self, action_ref: ActionRef) -> RawSnapshot: ...
+
+    def end_turn(self) -> RawSnapshot: ...
+
+    def close(self) -> None: ...
+
+
+class _InteractiveBot:
+    def __init__(self, session: "LiveFreecivSession"):
+        from freecivbot.bot.base_bot import BaseBot
+
+        class InteractiveBotImpl(BaseBot):
+            def __init__(self, owner: "LiveFreecivSession"):
+                super().__init__()
+                self._owner = owner
+
+            def conduct_turn(self, pplayer, info_controls, end_turn_hook):
+                super().conduct_turn(pplayer, info_controls, end_turn_hook)
+                self._publish_snapshot()
+
+            def calculate_next_move(self):
+                if self._turn_active:
+                    self._publish_snapshot()
+
+            def _publish_snapshot(self):
+                self._acquire_state()
+                self._owner._publish_snapshot(
+                    RawSnapshot(
+                        turn=self.turn,
+                        state=self._turn_state,
+                        actions=self._turn_opts,
+                    )
+                )
+
+        self.impl = InteractiveBotImpl(session)
+
+
+class _ConfiguredCivClient:
+    def __init__(self, bot, user_name: str, *, client_port: int, visual_monitor: bool = False):
+        from freecivbot.civclient import CivClient
+
+        class ConfiguredCivClientImpl(CivClient):
+            def init_control(self, ws_client):
+                self.ws_client = ws_client
+                self.init_controller()
+                if self.visual_monitor:
+                    self.monitor.start_monitor()
+                login_message = {
+                    "pid": 4,
+                    "username": self.user_name,
+                    "capability": "+Freeciv.Web.Devel-3.2",
+                    "version_label": "-dev",
+                    "major_version": 3,
+                    "minor_version": 1,
+                    "patch_version": 90,
+                    "port": self.client_port,
+                    "password": None,
+                    "subject": None,
+                }
+                self.ws_client.send(login_message)
+
+            def handle_chat_msg(self, packet):
+                from freecivbot.utils.fc_events import E_UNDEFINED
+
+                message = packet["message"]
+                conn_id = packet["conn_id"]
+                event = packet["event"]
+
+                if message is None:
+                    return
+                if event is None or event < 0 or event >= E_UNDEFINED:
+                    print("Undefined message event type")
+                    print(packet)
+                    print("\r\n")
+                    packet["event"] = event = E_UNDEFINED
+
+                if conn_id in self.clstate.connections:
+                    message = "<b>" + self.clstate.connections[conn_id]["username"] + ":</b>" + message
+                else:
+                    if "/metamessage" in message:
+                        return
+                    if "Metaserver message string" in message:
+                        return
+
+                packet["message"] = message
+                print(packet)
+                print("\r\n")
+
+                if "You are logged in as" in message:
+                    self.ws_client.send_message("/set minplayers 1")
+                    self.prepare_game()
+
+            def handle_conn_info(self, packet):
+                from freecivbot.connectivity.client_state import C_S_PREPARING
+                from freecivbot.utils.freecivlog import freelog
+
+                pconn = self.clstate.find_conn_by_id(packet["id"])
+
+                if not packet["used"]:
+                    if pconn is None:
+                        freelog(f"Server removed unknown connection {packet['id']}")
+                        return
+                    self.clstate.client_remove_cli_conn(pconn)
+                    pconn = None
+                else:
+                    pplayer = self.player_ctrl.valid_player_by_number(packet["player_num"])
+                    if pplayer is None:
+                        return
+                    packet["playing"] = pplayer
+
+                    if self.clstate.has_id(packet["id"]):
+                        self.clstate.init_state(packet)
+
+                    self.clstate.conn_list_append(packet)
+
+                if self.clstate.has_id(packet["id"]) and self.clstate.cur_player() != packet["playing"]:
+                    self.clstate.set_client_state(C_S_PREPARING)
+
+        self.impl = ConfiguredCivClientImpl(
+            bot,
+            user_name,
+            client_port=client_port,
+            visual_monitor=visual_monitor,
+        )
+
+
+class _ConfiguredCivConnection:
+    def __init__(self, civ_client, base_url: str, *, owner: "LiveFreecivSession", wait_for_server: int = 120, retry_interval: int = 5):
+        from math import ceil
+
+        import websocket
+
+        self._websocket = websocket
+        self.client = civ_client
+        self.base_url = base_url
+        self._owner = owner
+        self._loop = None
+        self._owner._connection = self
+        self.civserverport = self._reserve_client_port(base_url, civ_client.client_port)
+        self.client.client_port = self.civserverport
+        self.proxyport = 1000 + self.civserverport
+        self._retry_interval = retry_interval
+        self._num_retries = int(ceil(wait_for_server / retry_interval))
+        self._cur_retry = 0
+        self._ws_url = self._build_ws_url(base_url)
+        self.network_init()
+
+    def _build_ws_url(self, base_url: str) -> str:
+        parsed = urlparse(base_url)
+        scheme = "wss" if parsed.scheme == "https" else "ws"
+        host = parsed.hostname or "localhost"
+        port = parsed.port
+        if port is None:
+            port = 443 if scheme == "wss" else 80
+        return f"{scheme}://{host}:{port}/civsocket/{self.proxyport}"
+
+    def _reserve_client_port(self, base_url: str, requested_port: int) -> int:
+        parsed = urlparse(base_url)
+        scheme = parsed.scheme or "http"
+        host = parsed.hostname or "localhost"
+        port = parsed.port
+        if port is None:
+            port = 443 if scheme == "https" else 80
+        query = urlencode({"civserverport": requested_port})
+        launcher_url = f"{scheme}://{host}:{port}/civclientlauncher?{query}"
+        request = Request(launcher_url, method="POST")
+        with urlopen(request, timeout=10) as response:
+            result = response.headers.get("result")
+            reserved_port = response.headers.get("port")
+        if result != "success" or reserved_port is None:
+            raise RuntimeError(f"failed to reserve freeciv client port via {launcher_url}")
+        return int(reserved_port)
+
+    def _retry(self):
+        self._cur_retry += 1
+        time.sleep(self._retry_interval)
+        return self._detect_server_up()
+
+    def _detect_server_up(self):
+        ws = self._websocket.WebSocket()
+        try:
+            ws.connect(self._ws_url, timeout=10)
+            return True
+        except Exception as err:
+            print("Connect not successful:", err, " retrying in %s seconds." % self._retry_interval)
+            if self._cur_retry < self._num_retries:
+                return self._retry()
+            return False
+        finally:
+            try:
+                ws.close()
+            except Exception:
+                pass
+
+    def network_init(self):
+        self._cur_retry = 0
+        print("Connecting to server at %s ..." % self.base_url)
+        if self._detect_server_up():
+            self.websocket_init()
+        else:
+            print("Connection could not be established!")
+
+    def websocket_init(self):
+        from tornado import ioloop
+
+        from freecivbot.connectivity.clinet import CivWSClient
+
+        asyncio.set_event_loop(asyncio.new_event_loop())
+        ioloop.IOLoop.clear_current()
+        self._loop = ioloop.IOLoop.current()
+
+        client = CivWSClient(self.client)
+
+        def send_json(data):
+            if not client._ws_connection:
+                raise RuntimeError("Web socket connection is closed.")
+            msg = json.dumps(data, separators=(",", ":"))
+            client._ws_connection.write_message(msg)
+
+        client.send = send_json
+        client.connect(self._ws_url)
+
+        try:
+            self._loop.start()
+        except KeyboardInterrupt:
+            client.close()
+
+    def submit(self, fn) -> None:
+        if self._loop is None:
+            raise RuntimeError("freeciv connection loop is not ready")
+        done = threading.Event()
+        error: BaseException | None = None
+
+        def run():
+            nonlocal error
+            try:
+                fn()
+            except BaseException as exc:
+                error = exc
+            finally:
+                done.set()
+
+        self._loop.add_callback(run)
+        if not done.wait(timeout=10):
+            raise TimeoutError("timed out dispatching action to freeciv loop")
+        if error is not None:
+            raise error
+
+    def close(self) -> None:
+        if self._loop is None:
+            return
+        self.submit(self.client.close)
+
+
+class LiveFreecivSession:
+    def __init__(
+        self,
+        *,
+        username: str = "openenvbot",
+        client_port: int = 6000,
+        base_url: str = "http://localhost",
+        turn_timeout_s: float = 60.0,
+    ):
+        self.username = username
+        self.client_port = client_port
+        self.base_url = base_url
+        self.turn_timeout_s = turn_timeout_s
+
+        self._bot_wrapper: _InteractiveBot | None = None
+        self._client = None
+        self._connection: _ConfiguredCivConnection | None = None
+        self._thread: threading.Thread | None = None
+        self._ready = threading.Event()
+        self._snapshot_lock = threading.Lock()
+        self._snapshot: RawSnapshot | None = None
+        self._thread_error: BaseException | None = None
+        self._reset_counter = 0
+        self._session_seed = time.monotonic_ns() % 1_000_000
+
+    def reset(self, seed: int | None = None) -> RawSnapshot:
+        del seed
+        self.close()
+        self._reset_counter += 1
+        username = self._next_username()
+        client_port = self.client_port + ((self._session_seed + self._reset_counter - 1) % 3)
+
+        self._ready.clear()
+        self._thread_error = None
+        self._snapshot = None
+
+        self._bot_wrapper = _InteractiveBot(self)
+        self._client = _ConfiguredCivClient(
+            self._bot_wrapper.impl,
+            username,
+            client_port=client_port,
+            visual_monitor=False,
+        ).impl
+
+        def run() -> None:
+            try:
+                _ConfiguredCivConnection(self._client, self.base_url, owner=self)
+            except BaseException as exc:  # pragma: no cover - surfaced in waiters
+                self._thread_error = exc
+                self._ready.set()
+
+        self._thread = threading.Thread(target=run, name="freeciv-live-session", daemon=True)
+        self._thread.start()
+        return self._wait_for_snapshot("reset")
+
+    def apply_action(self, action_ref: ActionRef) -> RawSnapshot:
+        snapshot = self._require_snapshot()
+        action_list = snapshot.actions[action_ref.controller]
+        valid_actions = action_list.get_actions(action_ref.actor_id, valid_only=True)
+        action = None if valid_actions is None else valid_actions.get(action_ref.raw_action_key)
+        if action is None:
+            raise ValueError(
+                f"action {action_ref.raw_action_key} is no longer valid for {action_ref.controller}:{action_ref.actor_id}"
+            )
+        self._ready.clear()
+        connection = self._require_connection()
+        connection.submit(lambda: action_list.trigger_validated_action(action))
+        return self._wait_for_snapshot(action_ref.raw_action_key)
+
+    def end_turn(self) -> RawSnapshot:
+        if self._bot_wrapper is None:
+            raise RuntimeError("session has not been reset")
+        self._ready.clear()
+        connection = self._require_connection()
+        connection.submit(self._bot_wrapper.impl.end_turn)
+        return self._wait_for_snapshot("end_turn")
+
+    def close(self) -> None:
+        if self._connection is not None:
+            try:
+                self._connection.close()
+            except Exception:
+                pass
+        elif self._client is not None:
+            try:
+                self._client.close()
+            except Exception:
+                pass
+        if self._thread is not None and self._thread.is_alive():
+            self._thread.join(timeout=5)
+        self._bot_wrapper = None
+        self._client = None
+        self._connection = None
+        self._thread = None
+        self._snapshot = None
+        self._thread_error = None
+        self._ready.clear()
+
+    def _publish_snapshot(self, snapshot: RawSnapshot) -> None:
+        with self._snapshot_lock:
+            self._snapshot = snapshot
+        self._ready.set()
+
+    def _next_username(self) -> str:
+        suffix = str(self._session_seed + self._reset_counter)
+        prefix_len = max(1, 31 - len(suffix))
+        return f"{self.username[:prefix_len]}{suffix}"
+
+    def _require_connection(self) -> _ConfiguredCivConnection:
+        if self._connection is None:
+            raise RuntimeError("freeciv connection is not ready")
+        return self._connection
+
+    def _require_snapshot(self) -> RawSnapshot:
+        with self._snapshot_lock:
+            if self._snapshot is None:
+                raise RuntimeError("no live snapshot is available")
+            return self._snapshot
+
+    def _wait_for_snapshot(self, reason: str) -> RawSnapshot:
+        deadline = time.monotonic() + self.turn_timeout_s
+        while time.monotonic() < deadline:
+            if self._thread_error is not None:
+                raise RuntimeError(f"freeciv session failed during {reason}") from self._thread_error
+            if self._ready.wait(timeout=0.1):
+                if self._thread_error is not None:
+                    raise RuntimeError(f"freeciv session failed during {reason}") from self._thread_error
+                snapshot = self._require_snapshot()
+                if snapshot is not None:
+                    return snapshot
+        raise TimeoutError(f"timed out waiting for freeciv snapshot during {reason}")

build/lib/freeciv_env/server/__init__.py

@@ -0,0 +1,3 @@
+from freeciv_env.server.freeciv_environment import FreecivEnvironment
+
+__all__ = ["FreecivEnvironment"]

build/lib/freeciv_env/server/app.py

@@ -0,0 +1,42 @@
+from __future__ import annotations
+
+import os
+
+from openenv.core.env_server import create_app
+
+from freeciv_env.models import FreecivAction, FreecivObservation
+from freeciv_env.runtime import LiveFreecivSession
+from freeciv_env.server.freeciv_environment import FreecivEnvironment
+
+
+def create_live_session() -> LiveFreecivSession:
+    return LiveFreecivSession(
+        username=os.getenv("FREECIV_USERNAME", "openenvbot"),
+        client_port=int(os.getenv("FREECIV_CLIENT_PORT", "6000")),
+        base_url=os.getenv("FREECIV_SERVER_URL", "http://localhost"),
+        turn_timeout_s=float(os.getenv("FREECIV_TURN_TIMEOUT_S", "60")),
+    )
+
+
+def create_freeciv_app(*, session_factory=create_live_session, max_turns: int | None = None):
+    if max_turns is None:
+        max_turns = int(os.getenv("FREECIV_MAX_TURNS", "50"))
+    return create_app(
+        lambda: FreecivEnvironment(session_factory=session_factory, max_turns=max_turns),
+        FreecivAction,
+        FreecivObservation,
+        env_name="freeciv_env",
+    )
+
+
+app = create_freeciv_app()
+
+
+def main() -> None:
+    import uvicorn
+
+    uvicorn.run(app, host="0.0.0.0", port=8000, ws_ping_interval=300, ws_ping_timeout=300)
+
+
+if __name__ == "__main__":
+    main()

build/lib/freeciv_env/server/freeciv_environment.py

@@ -0,0 +1,163 @@
+from __future__ import annotations
+
+from typing import Callable
+from uuid import uuid4
+
+from openenv.core.env_server.interfaces import Environment
+
+from freeciv_env.adapter import (
+    ActionLookupKey,
+    ActionRef,
+    PreparedObservation,
+    RawSnapshot,
+    SnapshotMetrics,
+    action_lookup_key,
+    prepare_observation,
+)
+from freeciv_env.models import FreecivAction, FreecivObservation, FreecivState
+from freeciv_env.runtime import FreecivSession
+
+
+class FreecivEnvironment(Environment[FreecivAction, FreecivObservation, FreecivState]):
+    SUPPORTS_CONCURRENT_SESSIONS = False
+
+    def __init__(self, session_factory: Callable[[], FreecivSession], max_turns: int = 50):
+        super().__init__()
+        self._session_factory = session_factory
+        self.max_turns = max_turns
+        self._session: FreecivSession | None = None
+        self._snapshot: RawSnapshot | None = None
+        self._metrics: SnapshotMetrics | None = None
+        self._action_refs: dict[ActionLookupKey, ActionRef] = {}
+        self._state = FreecivState(episode_id=str(uuid4()), step_count=0)
+
+    def reset(
+        self,
+        seed: int | None = None,
+        episode_id: str | None = None,
+        **kwargs,
+    ) -> FreecivObservation:
+        del kwargs
+        self.close()
+        self._session = self._session_factory()
+        snapshot = self._session.reset(seed=seed)
+        prepared = prepare_observation(
+            snapshot,
+            reward=0.0,
+            done=self._is_done(snapshot),
+            status="ready",
+            metadata={},
+        )
+        self._commit(snapshot, prepared, episode_id=episode_id or str(uuid4()))
+        return prepared.observation
+
+    def step(
+        self,
+        action: FreecivAction,
+        timeout_s: float | None = None,
+        **kwargs,
+    ) -> FreecivObservation:
+        del timeout_s, kwargs
+        if self._session is None or self._snapshot is None or self._metrics is None:
+            raise RuntimeError("environment must be reset before step")
+
+        self._state.step_count += 1
+        if action.action_type == "end_turn":
+            next_snapshot = self._session.end_turn()
+            reward = self._reward_for_transition(action, self._metrics, next_snapshot)
+            prepared = prepare_observation(
+                next_snapshot,
+                reward=reward,
+                done=self._is_done(next_snapshot),
+                status="ok",
+                metadata={},
+            )
+            self._commit(next_snapshot, prepared, episode_id=self._state.episode_id)
+            return prepared.observation
+
+        ref = self._action_refs.get(action_lookup_key(action))
+        if ref is None:
+            prepared = prepare_observation(
+                self._snapshot,
+                reward=-0.25,
+                done=self._is_done(self._snapshot),
+                status="invalid_action",
+                metadata={"error": "action is not currently legal"},
+            )
+            self._commit(self._snapshot, prepared, episode_id=self._state.episode_id, replace_snapshot=False)
+            return prepared.observation
+
+        next_snapshot = self._session.apply_action(ref)
+        reward = self._reward_for_transition(action, self._metrics, next_snapshot)
+        prepared = prepare_observation(
+            next_snapshot,
+            reward=reward,
+            done=self._is_done(next_snapshot),
+            status="ok",
+            metadata={},
+        )
+        self._commit(next_snapshot, prepared, episode_id=self._state.episode_id)
+        return prepared.observation
+
+    @property
+    def state(self) -> FreecivState:
+        return self._state
+
+    def close(self) -> None:
+        if self._session is not None:
+            self._session.close()
+        self._session = None
+        self._snapshot = None
+        self._metrics = None
+        self._action_refs = {}
+
+    def _commit(
+        self,
+        snapshot: RawSnapshot,
+        prepared: PreparedObservation,
+        *,
+        episode_id: str,
+        replace_snapshot: bool = True,
+    ) -> None:
+        if replace_snapshot:
+            self._snapshot = snapshot
+        self._metrics = prepared.metrics
+        self._action_refs = prepared.action_refs
+        self._state = FreecivState(
+            episode_id=episode_id,
+            step_count=self._state.step_count,
+            turn=prepared.observation.turn,
+            score=prepared.observation.score,
+            known_tiles=prepared.observation.known_tiles,
+            visible_tiles=prepared.observation.visible_tiles,
+            city_count=prepared.observation.city_count,
+            unit_count=prepared.observation.unit_count,
+            techs_researched=prepared.observation.techs_researched,
+        )
+
+    def _reward_for_transition(
+        self,
+        action: FreecivAction,
+        previous: SnapshotMetrics,
+        next_snapshot: RawSnapshot,
+    ) -> float:
+        from freeciv_env.adapter import extract_metrics
+
+        current = extract_metrics(next_snapshot)
+        reward = {
+            "end_turn": 0.0,
+            "move_unit": 0.01,
+            "build_city": 0.10,
+            "set_city_production": 0.05,
+            "set_research": 0.05,
+        }[action.action_type]
+        reward += max(current.score - previous.score, 0.0) * 0.02
+        reward += max(current.known_tiles - previous.known_tiles, 0) * 0.01
+        reward += max(current.city_count - previous.city_count, 0) * 0.50
+        reward += max(current.techs_researched - previous.techs_researched, 0) * 0.25
+        return float(reward)
+
+    def _is_done(self, snapshot: RawSnapshot) -> bool:
+        player = snapshot.state.get("player", {})
+        alive = bool(player.get("my_is_alive", True))
+        return (not alive) or snapshot.turn >= self.max_turns

build/lib/server/app.py

@@ -0,0 +1,10 @@
+from freeciv_env.server.app import app as app
+from freeciv_env.server.app import main as _main
+
+
+def main() -> None:
+    _main()
+
+
+if __name__ == "__main__":
+    main()

client.py

@@ -0,0 +1 @@
+from freeciv_env.client import *

freeciv_env.egg-info/PKG-INFO

@@ -0,0 +1,136 @@
+Metadata-Version: 2.4
+Name: freeciv-env
+Version: 0.1.0
+Summary: OpenEnv environment for Freeciv via freeciv-bot
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+Requires-Dist: openenv-core[core]==0.2.1
+Requires-Dist: freecivbot @ git+https://github.com/chris1869/freeciv-bot.git
+Requires-Dist: uvicorn>=0.35.0
+Provides-Extra: dev
+Requires-Dist: pytest>=8.4.1; extra == "dev"
+Requires-Dist: requests>=2.32.5; extra == "dev"
+Provides-Extra: train
+Requires-Dist: accelerate>=1.10.0; extra == "train"
+Requires-Dist: bitsandbytes>=0.47.0; extra == "train"
+Requires-Dist: datasets>=4.0.0; extra == "train"
+Requires-Dist: trl>=0.24.0; extra == "train"
+Requires-Dist: unsloth>=2026.3.4; extra == "train"
+
+---
+title: Freeciv Environment Server
+emoji: 🎮
+colorFrom: blue
+colorTo: indigo
+sdk: docker
+pinned: false
+app_port: 8000
+base_path: /web
+tags:
+  - openenv
+---
+
+# freeciv-env
+
+OpenEnv environment for Freeciv, built on top of `freeciv-bot`.
+
+## Current scope
+
+This environment exposes a small, trainable action surface:
+
+- `end_turn`
+- `move_unit(unit_id, direction)`
+- `build_city(unit_id)`
+- `set_city_production(city_id, target)`
+- `set_research(tech_name)`
+
+Observations are text-first and include compact structured summaries of:
+
+- current turn
+- score
+- known and visible map tiles
+- units
+- cities
+- legal actions
+
+## Local development
+
+Install dependencies:
+
+```bash
+uv sync --extra dev
+```
+
+Run tests:
+
+```bash
+uv run pytest
+```
+
+Run the server:
+
+```bash
+uv run uvicorn freeciv_env.server.app:app --host 0.0.0.0 --port 8000
+```
+
+Run the fast GRPO loop:
+
+```bash
+uv sync --extra dev --extra train
+uv run python scripts/train_grpo_fast.py --env-url http://127.0.0.1 --max-steps 50
+```
+
+## Hackathon / Unsloth notes
+
+For the hackathon Colab submission path on H100s, Unsloth recommended the BF16 OpenEnv gpt-oss 20B notebook:
+
+- <https://colab.research.google.com/github/unslothai/notebooks/blob/main/nb/OpenEnv_gpt_oss_(20B)_Reinforcement_Learning_2048_Game_BF16.ipynb>
+
+If you adapt that notebook for this environment, reduce `max_steps` to `300` for a faster run.
+
+Useful notebook indexes:
+
+- RL notebooks: <https://unsloth.ai/docs/get-started/unsloth-notebooks#grpo-reasoning-rl>
+- all notebooks: <https://unsloth.ai/docs/get-started/unsloth-notebooks>
+- notebook repo: <https://github.com/unslothai/notebooks/tree/main/nb>
+
+If GRPO is too slow, start from a smaller notebook with `fast_inference = True` and add the Freeciv/OpenEnv calls:
+
+- <https://colab.research.google.com/github/unslothai/notebooks/blob/main/nb/Qwen3_(4B)-GRPO.ipynb>
+- <https://colab.research.google.com/github/unslothai/notebooks/blob/main/nb/Advanced_Llama3_2_(3B)_GRPO_LoRA.ipynb>
+
+If vLLM GRPO fails, Unsloth suggested a clean virtualenv install:
+
+```bash
+python -m venv unsloth_env
+source unsloth_env/bin/activate
+pip install --upgrade pip && pip install uv
+uv pip install unsloth vllm --torch-backend=auto
+```
+
+If Unsloth is already installed, update it for the latest GRPO fixes:
+
+```bash
+pip install --upgrade --no-cache-dir --no-deps unsloth unsloth_zoo
+```
+
+## Live runtime requirements
+
+The default server app uses `freeciv-bot` against a local Freeciv Web runtime.
+
+Environment variables:
+
+- `FREECIV_SERVER_URL` (default: `http://127.0.0.1`)
+- `FREECIV_USERNAME` (default: `openenvbot`)
+- `FREECIV_CLIENT_PORT` (default: `6000`)
+- `FREECIV_TURN_TIMEOUT_S` (default: `60`)
+
+The included automated tests use a fake session backend, so they do not require a live Freeciv server.
+
+The GRPO training script uses:
+
+- `Qwen/Qwen3.5-0.8B`
+- Unsloth bf16 LoRA loading
+- TRL `GRPOTrainer`
+- integer-only action selection to minimize generated tokens
+- offline GRPO over env-sampled states for maximum throughput

freeciv_env.egg-info/SOURCES.txt

@@ -0,0 +1,23 @@
+README.md
+pyproject.toml
+freeciv_env/__init__.py
+freeciv_env/adapter.py
+freeciv_env/client.py
+freeciv_env/grpo.py
+freeciv_env/models.py
+freeciv_env/runtime.py
+freeciv_env.egg-info/PKG-INFO
+freeciv_env.egg-info/SOURCES.txt
+freeciv_env.egg-info/dependency_links.txt
+freeciv_env.egg-info/entry_points.txt
+freeciv_env.egg-info/requires.txt
+freeciv_env.egg-info/top_level.txt
+freeciv_env/server/__init__.py
+freeciv_env/server/app.py
+freeciv_env/server/freeciv_environment.py
+server/__init__.py
+server/app.py
+tests/test_adapter.py
+tests/test_environment.py
+tests/test_grpo_utils.py
+tests/test_server_roundtrip.py

freeciv_env.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

freeciv_env.egg-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+server = server.app:main

freeciv_env.egg-info/requires.txt

@@ -0,0 +1,14 @@
+openenv-core[core]==0.2.1
+freecivbot @ git+https://github.com/chris1869/freeciv-bot.git
+uvicorn>=0.35.0
+
+[dev]
+pytest>=8.4.1
+requests>=2.32.5
+
+[train]
+accelerate>=1.10.0
+bitsandbytes>=0.47.0
+datasets>=4.0.0
+trl>=0.24.0
+unsloth>=2026.3.4

freeciv_env.egg-info/top_level.txt

@@ -0,0 +1,2 @@
+freeciv_env
+server

freeciv_env/__init__.py

@@ -0,0 +1,10 @@
+from freeciv_env.client import FreecivEnv
+from freeciv_env.models import FreecivAction, FreecivObservation, FreecivState, LegalAction
+
+__all__ = [
+    "FreecivAction",
+    "FreecivEnv",
+    "FreecivObservation",
+    "FreecivState",
+    "LegalAction",
+]

freeciv_env/adapter.py

@@ -0,0 +1,335 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any
+
+from freeciv_env.models import CitySummary, FreecivAction, FreecivObservation, LegalAction, UnitSummary
+
+
+ActionLookupKey = tuple[str, int | None, int | None, str | None]
+
+
+@dataclass(frozen=True)
+class ActionRef:
+    controller: str
+    actor_id: int | str
+    raw_action_key: str
+
+
+@dataclass
+class RawSnapshot:
+    turn: int
+    state: dict[str, Any]
+    actions: dict[str, Any]
+
+
+@dataclass(frozen=True)
+class SnapshotMetrics:
+    score: float
+    known_tiles: int
+    visible_tiles: int
+    city_count: int
+    unit_count: int
+    techs_researched: int
+
+
+@dataclass
+class PreparedObservation:
+    observation: FreecivObservation
+    metrics: SnapshotMetrics
+    action_refs: dict[ActionLookupKey, ActionRef]
+
+
+def _map_status_rows(raw_state: dict[str, Any]) -> list[list[int | float]]:
+    raw_map = raw_state.get("map", {})
+    status = raw_map.get("status", [])
+    return status if isinstance(status, list) else []
+
+
+def count_known_tiles(raw_state: dict[str, Any]) -> int:
+    return sum(1 for row in _map_status_rows(raw_state) for value in row if value and value > 0)
+
+
+def count_visible_tiles(raw_state: dict[str, Any]) -> int:
+    return sum(1 for row in _map_status_rows(raw_state) for value in row if value and value >= 2)
+
+
+def extract_metrics(snapshot: RawSnapshot) -> SnapshotMetrics:
+    player = snapshot.state.get("player", {})
+    return SnapshotMetrics(
+        score=float(player.get("my_score", 0.0)),
+        known_tiles=count_known_tiles(snapshot.state),
+        visible_tiles=count_visible_tiles(snapshot.state),
+        city_count=len(snapshot.state.get("city", {})),
+        unit_count=len(snapshot.state.get("unit", {})),
+        techs_researched=int(player.get("my_techs_researched", 0) or 0),
+    )
+
+
+def action_lookup_key(action: FreecivAction) -> ActionLookupKey:
+    if action.action_type == "move_unit":
+        return ("move_unit", action.unit_id, action.direction, None)
+    if action.action_type == "build_city":
+        return ("build_city", action.unit_id, None, None)
+    if action.action_type == "set_city_production":
+        return ("set_city_production", action.city_id, None, action.target)
+    if action.action_type == "set_research":
+        return ("set_research", None, None, action.target)
+    return ("end_turn", None, None, None)
+
+
+def _parse_target_name(raw_action_key: str, prefix: str) -> str:
+    suffix = raw_action_key.removeprefix(prefix)
+    name, _sep, _tail = suffix.rpartition("_")
+    return name or suffix
+
+
+
+def _controller_actions(snapshot: RawSnapshot, controller: str) -> dict[str, Any]:
+    raw_actions = snapshot.actions.get(controller, {})
+    if isinstance(raw_actions, dict):
+        return raw_actions
+    if hasattr(raw_actions, "json_struct"):
+        json_actions = raw_actions.json_struct()
+        return json_actions if isinstance(json_actions, dict) else {}
+    return {}
+
+
+
+def _extract_legal_actions(snapshot: RawSnapshot) -> tuple[list[LegalAction], dict[ActionLookupKey, ActionRef]]:
+    legal_actions: list[LegalAction] = [
+        LegalAction(
+            action_type="end_turn",
+            label="End the current turn",
+            raw_action_key="__end_turn__",
+        )
+    ]
+    refs: dict[ActionLookupKey, ActionRef] = {}
+
+    for actor_id, action_map in _controller_actions(snapshot, "unit").items():
+        unit_id = int(actor_id)
+        if action_map.get("build"):
+            legal_actions.append(
+                LegalAction(
+                    action_type="build_city",
+                    label=f"Build a city with unit {unit_id}",
+                    unit_id=unit_id,
+                    raw_action_key="build",
+                )
+            )
+            refs[("build_city", unit_id, None, None)] = ActionRef(
+                controller="unit",
+                actor_id=unit_id,
+                raw_action_key="build",
+            )
+        for raw_action_key, enabled in sorted(action_map.items()):
+            if not enabled or not raw_action_key.startswith("goto_"):
+                continue
+            direction = int(raw_action_key.split("_", 1)[1])
+            legal_actions.append(
+                LegalAction(
+                    action_type="move_unit",
+                    label=f"Move unit {unit_id} in direction {direction}",
+                    unit_id=unit_id,
+                    direction=direction,
+                    raw_action_key=raw_action_key,
+                )
+            )
+            refs[("move_unit", unit_id, direction, None)] = ActionRef(
+                controller="unit",
+                actor_id=unit_id,
+                raw_action_key=raw_action_key,
+            )
+
+    for actor_id, action_map in _controller_actions(snapshot, "city").items():
+        city_id = int(actor_id)
+        for raw_action_key, enabled in sorted(action_map.items()):
+            if not enabled:
+                continue
+            if raw_action_key.startswith("change_unit_prod_"):
+                target = _parse_target_name(raw_action_key, "change_unit_prod_")
+            elif raw_action_key.startswith("change_improve_prod_"):
+                target = _parse_target_name(raw_action_key, "change_improve_prod_")
+            else:
+                continue
+            legal_actions.append(
+                LegalAction(
+                    action_type="set_city_production",
+                    label=f"Set city {city_id} production to {target}",
+                    city_id=city_id,
+                    target=target,
+                    raw_action_key=raw_action_key,
+                )
+            )
+            refs[("set_city_production", city_id, None, target)] = ActionRef(
+                controller="city",
+                actor_id=city_id,
+                raw_action_key=raw_action_key,
+            )
+
+    tech_actions = _controller_actions(snapshot, "tech").get("cur_player", {})
+    for raw_action_key, enabled in sorted(tech_actions.items()):
+        if not enabled or not raw_action_key.startswith("research_tech_"):
+            continue
+        target = _parse_target_name(raw_action_key, "research_tech_")
+        legal_actions.append(
+            LegalAction(
+                action_type="set_research",
+                label=f"Research {target}",
+                target=target,
+                raw_action_key=raw_action_key,
+            )
+        )
+        refs[("set_research", None, None, target)] = ActionRef(
+            controller="tech",
+            actor_id="cur_player",
+            raw_action_key=raw_action_key,
+        )
+
+    legal_actions.sort(
+        key=lambda item: (
+            item.action_type,
+            item.unit_id or -1,
+            item.city_id or -1,
+            item.direction or -1,
+            item.target or "",
+        )
+    )
+    return legal_actions, refs
+
+
+def _extract_unit_summaries(snapshot: RawSnapshot) -> list[UnitSummary]:
+    unit_actions = _controller_actions(snapshot, "unit")
+    units: list[UnitSummary] = []
+    for actor_id, unit in sorted(snapshot.state.get("unit", {}).items(), key=lambda item: int(item[0])):
+        action_map = unit_actions.get(str(actor_id), unit_actions.get(actor_id, {}))
+        move_directions = sorted(
+            int(raw_action_key.split("_", 1)[1])
+            for raw_action_key, enabled in action_map.items()
+            if enabled and raw_action_key.startswith("goto_")
+        )
+        units.append(
+            UnitSummary(
+                unit_id=int(actor_id),
+                unit_type=str(unit.get("type_rule_name", "Unknown")),
+                health=int(unit.get("health", 0) or 0),
+                moves_left=int(unit.get("moves_left", unit.get("movesleft", 0)) or 0),
+                home_city_id=(
+                    int(unit.get("home_city"))
+                    if unit.get("home_city") not in (None, -1, "")
+                    else None
+                ),
+                veteran_level=int(unit.get("veteran", 0) or 0),
+                can_build_city=bool(action_map.get("build", False)),
+                move_directions=move_directions,
+            )
+        )
+    return units
+
+
+def _extract_city_summaries(snapshot: RawSnapshot) -> list[CitySummary]:
+    city_actions = _controller_actions(snapshot, "city")
+    cities: list[CitySummary] = []
+    for actor_id, city in sorted(snapshot.state.get("city", {}).items(), key=lambda item: int(item[0])):
+        action_map = city_actions.get(str(actor_id), city_actions.get(actor_id, {}))
+        production_options = [
+            _parse_target_name(raw_action_key, "change_unit_prod_")
+            for raw_action_key, enabled in sorted(action_map.items())
+            if enabled and raw_action_key.startswith("change_unit_prod_")
+        ] + [
+            _parse_target_name(raw_action_key, "change_improve_prod_")
+            for raw_action_key, enabled in sorted(action_map.items())
+            if enabled and raw_action_key.startswith("change_improve_prod_")
+        ]
+        cities.append(
+            CitySummary(
+                city_id=int(actor_id),
+                size=int(city.get("size", 0) or 0),
+                prod_food=int(city.get("prod_food", 0) or 0),
+                prod_shield=int(city.get("prod_shield", 0) or 0),
+                prod_trade=int(city.get("prod_trade", 0) or 0),
+                surplus_food=int(city.get("surplus_food", 0) or 0),
+                surplus_shield=int(city.get("surplus_shield", 0) or 0),
+                surplus_trade=int(city.get("surplus_trade", 0) or 0),
+                production_kind=(
+                    int(city.get("production_kind"))
+                    if city.get("production_kind") is not None
+                    else None
+                ),
+                production_value=(
+                    int(city.get("production_value"))
+                    if city.get("production_value") is not None
+                    else None
+                ),
+                turns_to_complete=(
+                    float(city.get("turns_to_prod_complete"))
+                    if city.get("turns_to_prod_complete") is not None
+                    else None
+                ),
+                production_options=production_options,
+            )
+        )
+    return cities
+
+
+def _build_summary(
+    snapshot: RawSnapshot,
+    metrics: SnapshotMetrics,
+    units: list[UnitSummary],
+    cities: list[CitySummary],
+    legal_actions: list[LegalAction],
+) -> str:
+    player = snapshot.state.get("player", {})
+    lines = [
+        f"Turn {snapshot.turn}",
+        f"Score {metrics.score:.1f}",
+        f"Map: {metrics.known_tiles} known tiles, {metrics.visible_tiles} visible tiles",
+        f"Economy: {player.get('my_gold', 0)} gold, science rate {player.get('my_science', 0)}%",
+        f"Cities: {metrics.city_count}",
+    ]
+    for city in cities[:5]:
+        lines.append(
+            f"- City {city.city_id}: size {city.size}, food {city.prod_food}/{city.surplus_food:+d}, "
+            f"shields {city.prod_shield}/{city.surplus_shield:+d}, trade {city.prod_trade}/{city.surplus_trade:+d}"
+        )
+    lines.append(f"Units: {metrics.unit_count}")
+    for unit in units[:8]:
+        lines.append(
+            f"- Unit {unit.unit_id}: {unit.unit_type}, hp {unit.health}, moves_left {unit.moves_left}, "
+            f"build_city={str(unit.can_build_city).lower()}, move_dirs={unit.move_directions}"
+        )
+    lines.append(f"Techs researched: {metrics.techs_researched}")
+    lines.append(f"Legal actions exposed: {len(legal_actions)}")
+    return "\n".join(lines)
+
+
+def prepare_observation(
+    snapshot: RawSnapshot,
+    *,
+    reward: float,
+    done: bool,
+    status: str,
+    metadata: dict[str, Any] | None = None,
+) -> PreparedObservation:
+    legal_actions, action_refs = _extract_legal_actions(snapshot)
+    metrics = extract_metrics(snapshot)
+    units = _extract_unit_summaries(snapshot)
+    cities = _extract_city_summaries(snapshot)
+    observation = FreecivObservation(
+        turn=snapshot.turn,
+        score=metrics.score,
+        known_tiles=metrics.known_tiles,
+        visible_tiles=metrics.visible_tiles,
+        city_count=metrics.city_count,
+        unit_count=metrics.unit_count,
+        techs_researched=metrics.techs_researched,
+        status=status,
+        summary=_build_summary(snapshot, metrics, units, cities, legal_actions),
+        units=units,
+        cities=cities,
+        legal_actions=legal_actions,
+        reward=reward,
+        done=done,
+        metadata=metadata or {},
+    )
+    return PreparedObservation(observation=observation, metrics=metrics, action_refs=action_refs)

freeciv_env/client.py

@@ -0,0 +1,22 @@
+from __future__ import annotations
+
+from openenv.core.client_types import StepResult
+from openenv.core.env_client import EnvClient
+
+from freeciv_env.models import FreecivAction, FreecivObservation, FreecivState
+
+
+class FreecivEnv(EnvClient[FreecivAction, FreecivObservation, FreecivState]):
+    def _step_payload(self, action: FreecivAction) -> dict:
+        return action.model_dump(exclude_none=True)
+
+    def _parse_result(self, payload: dict) -> StepResult[FreecivObservation]:
+        observation = FreecivObservation(**payload["observation"])
+        return StepResult(
+            observation=observation,
+            reward=payload.get("reward"),
+            done=payload.get("done", False),
+        )
+
+    def _parse_state(self, payload: dict) -> FreecivState:
+        return FreecivState(**payload)

freeciv_env/grpo.py

@@ -0,0 +1,97 @@
+from __future__ import annotations
+
+import re
+from typing import Iterable
+
+from freeciv_env.models import FreecivAction, FreecivObservation, LegalAction
+
+SYSTEM_PROMPT = (
+    "You are choosing the next action for a Freeciv agent. "
+    "Return only the integer index of the best legal action. "
+    "Do not output words, punctuation, JSON, or explanations."
+)
+
+TASK_PROMPT = (
+    "Pick the legal action index that maximizes immediate reward. "
+    "Invalid actions are penalized. Shorter outputs are better."
+)
+
+
+def format_action_line(index: int, action: LegalAction) -> str:
+    return f"{index}: {action.label}"
+
+
+def build_turn_prompt(observation: FreecivObservation, task_prompt: str = TASK_PROMPT) -> str:
+    action_lines = [format_action_line(index, action) for index, action in enumerate(observation.legal_actions)]
+    return (
+        f"{task_prompt}\n\n"
+        f"State:\n{observation.summary}\n\n"
+        f"Legal actions:\n" + "\n".join(action_lines) + "\n\n"
+        "Return exactly one integer index."
+    )
+
+
+def parse_action_choice(completion_text: str, legal_actions: Iterable[LegalAction]) -> FreecivAction | None:
+    legal_actions = list(legal_actions)
+    match = re.search(r"-?\d+", completion_text)
+    if match is None:
+        return None
+    index = int(match.group(0))
+    if index < 0 or index >= len(legal_actions):
+        return None
+    action = legal_actions[index]
+    if action.action_type == "end_turn":
+        return FreecivAction(action_type="end_turn")
+    if action.action_type == "move_unit":
+        return FreecivAction(action_type="move_unit", unit_id=action.unit_id, direction=action.direction)
+    if action.action_type == "build_city":
+        return FreecivAction(action_type="build_city", unit_id=action.unit_id)
+    if action.action_type == "set_city_production":
+        return FreecivAction(action_type="set_city_production", city_id=action.city_id, target=action.target)
+    if action.action_type == "set_research":
+        return FreecivAction(action_type="set_research", target=action.target)
+    raise ValueError(f"unsupported action_type: {action.action_type}")
+
+
+def action_priority(action: LegalAction) -> tuple[int, int]:
+    if action.action_type == "build_city":
+        return (500, 0)
+    if action.action_type == "set_research":
+        return (400, 0)
+    if action.action_type == "set_city_production":
+        bonus = 50 if (action.target or "") == "Settlers" else 0
+        return (300 + bonus, 0)
+    if action.action_type == "move_unit":
+        return (200, -(action.direction or 0))
+    if action.action_type == "end_turn":
+        return (0, 0)
+    return (-1000, 0)
+
+
+
+def oracle_action_index(legal_actions: Iterable[LegalAction]) -> int:
+    legal_actions = list(legal_actions)
+    if not legal_actions:
+        raise ValueError("no legal actions available")
+    best_index = 0
+    best_priority = action_priority(legal_actions[0])
+    for index, action in enumerate(legal_actions[1:], start=1):
+        priority = action_priority(action)
+        if priority > best_priority:
+            best_index = index
+            best_priority = priority
+    return best_index
+
+
+
+def reward_from_oracle(completions, best_index, **kwargs):
+    del kwargs
+    rewards = []
+    for completion, expected in zip(completions, best_index):
+        match = re.search(r"-?\d+", completion if isinstance(completion, str) else str(completion))
+        if match is None:
+            rewards.append(-0.25)
+            continue
+        chosen = int(match.group(0))
+        rewards.append(1.0 if chosen == int(expected) else 0.0)
+    return rewards

freeciv_env/models.py

@@ -0,0 +1,112 @@
+from __future__ import annotations
+
+from typing import Literal
+
+from pydantic import BaseModel, Field, model_validator
+
+from openenv.core.env_server.types import Action, Observation, State
+
+
+class UnitSummary(BaseModel):
+    unit_id: int = Field(..., description="Freeciv unit id")
+    unit_type: str = Field(..., description="Ruleset unit type name")
+    health: int = Field(0, description="Current health")
+    moves_left: int = Field(0, description="Movement points remaining")
+    home_city_id: int | None = Field(None, description="Home city id, if any")
+    veteran_level: int = Field(0, description="Veteran level")
+    can_build_city: bool = Field(False, description="Whether the unit can found a city now")
+    move_directions: list[int] = Field(default_factory=list, description="Legal move direction indexes")
+
+
+class CitySummary(BaseModel):
+    city_id: int = Field(..., description="Freeciv city id")
+    size: int = Field(..., description="Population size")
+    prod_food: int = Field(0, description="Gross food output")
+    prod_shield: int = Field(0, description="Gross shield output")
+    prod_trade: int = Field(0, description="Gross trade output")
+    surplus_food: int = Field(0, description="Net food surplus")
+    surplus_shield: int = Field(0, description="Net shield surplus")
+    surplus_trade: int = Field(0, description="Net trade surplus")
+    production_kind: int | None = Field(None, description="Current production kind enum from Freeciv")
+    production_value: int | None = Field(None, description="Current production value id from Freeciv")
+    turns_to_complete: float | None = Field(None, description="Turns until current production completes")
+    production_options: list[str] = Field(default_factory=list, description="Legal production targets")
+
+
+class LegalAction(BaseModel):
+    action_type: Literal[
+        "end_turn",
+        "move_unit",
+        "build_city",
+        "set_city_production",
+        "set_research",
+    ]
+    label: str = Field(..., description="Human-readable action label")
+    unit_id: int | None = Field(None, description="Target unit id")
+    city_id: int | None = Field(None, description="Target city id")
+    direction: int | None = Field(None, description="Freeciv direction index 0..7")
+    target: str | None = Field(None, description="Production or tech target name")
+    raw_action_key: str | None = Field(None, description="Underlying freeciv-bot action key")
+
+
+class FreecivAction(Action):
+    action_type: Literal[
+        "end_turn",
+        "move_unit",
+        "build_city",
+        "set_city_production",
+        "set_research",
+    ]
+    unit_id: int | None = None
+    city_id: int | None = None
+    direction: int | None = None
+    target: str | None = None
+
+    @model_validator(mode="after")
+    def validate_shape(self) -> "FreecivAction":
+        if self.action_type == "end_turn":
+            return self
+        if self.action_type == "move_unit":
+            if self.unit_id is None or self.direction is None:
+                raise ValueError("move_unit requires unit_id and direction")
+            return self
+        if self.action_type == "build_city":
+            if self.unit_id is None:
+                raise ValueError("build_city requires unit_id")
+            return self
+        if self.action_type == "set_city_production":
+            if self.city_id is None or not self.target:
+                raise ValueError("set_city_production requires city_id and target")
+            return self
+        if self.action_type == "set_research":
+            if not self.target:
+                raise ValueError("set_research requires target")
+            return self
+        raise ValueError(f"unsupported action_type: {self.action_type}")
+
+
+class FreecivObservation(Observation):
+    turn: int = Field(..., description="Current game turn")
+    score: float = Field(..., description="Current player score")
+    known_tiles: int = Field(..., description="Tiles known to the player")
+    visible_tiles: int = Field(..., description="Tiles currently visible to the player")
+    city_count: int = Field(..., description="Number of owned cities")
+    unit_count: int = Field(..., description="Number of owned units")
+    techs_researched: int = Field(..., description="Number of researched techs")
+    status: str = Field("ok", description="High-level environment status")
+    summary: str = Field(..., description="Compact text summary for LLMs")
+    units: list[UnitSummary] = Field(default_factory=list, description="Compact unit summaries")
+    cities: list[CitySummary] = Field(default_factory=list, description="Compact city summaries")
+    legal_actions: list[LegalAction] = Field(default_factory=list, description="Legal actions exposed by the environment")
+    reward: float = Field(0.0, description="Reward from the last action")
+    done: bool = Field(False, description="Whether the episode is done")
+
+
+class FreecivState(State):
+    turn: int = Field(0, description="Current game turn")
+    score: float = Field(0.0, description="Current player score")
+    known_tiles: int = Field(0, description="Known tiles")
+    visible_tiles: int = Field(0, description="Visible tiles")
+    city_count: int = Field(0, description="Owned city count")
+    unit_count: int = Field(0, description="Owned unit count")
+    techs_researched: int = Field(0, description="Researched tech count")

freeciv_env/runtime.py

@@ -0,0 +1,432 @@
+from __future__ import annotations
+
+import asyncio
+import json
+import threading
+import time
+from collections import deque
+from typing import Protocol
+from urllib.parse import urlencode, urlparse
+from urllib.request import Request, urlopen
+
+from freeciv_env.adapter import ActionRef, RawSnapshot
+
+
+DEBUG_EVENTS: deque[str] = deque(maxlen=400)
+
+
+def debug_event(message: str) -> None:
+    DEBUG_EVENTS.append(f"{time.strftime('%H:%M:%S')} {message}")
+
+
+class FreecivSession(Protocol):
+    def reset(self, seed: int | None = None) -> RawSnapshot: ...
+
+    def apply_action(self, action_ref: ActionRef) -> RawSnapshot: ...
+
+    def end_turn(self) -> RawSnapshot: ...
+
+    def close(self) -> None: ...
+
+
+class _InteractiveBot:
+    def __init__(self, session: "LiveFreecivSession"):
+        from freecivbot.bot.base_bot import BaseBot
+
+        class InteractiveBotImpl(BaseBot):
+            def __init__(self, owner: "LiveFreecivSession"):
+                super().__init__()
+                self._owner = owner
+
+            def conduct_turn(self, pplayer, info_controls, end_turn_hook):
+                super().conduct_turn(pplayer, info_controls, end_turn_hook)
+                self._publish_snapshot()
+
+            def calculate_next_move(self):
+                if self._turn_active:
+                    self._publish_snapshot()
+
+            def _publish_snapshot(self):
+                self._acquire_state()
+                self._owner._publish_snapshot(
+                    RawSnapshot(
+                        turn=self.turn,
+                        state=self._turn_state,
+                        actions=self._turn_opts,
+                    )
+                )
+
+        self.impl = InteractiveBotImpl(session)
+
+
+class _ConfiguredCivClient:
+    def __init__(self, bot, user_name: str, *, client_port: int, visual_monitor: bool = False):
+        from freecivbot.civclient import CivClient
+
+        class ConfiguredCivClientImpl(CivClient):
+            def init_control(self, ws_client):
+                self.ws_client = ws_client
+                self.init_controller()
+                if self.visual_monitor:
+                    self.monitor.start_monitor()
+                login_message = {
+                    "pid": 4,
+                    "username": self.user_name,
+                    "capability": "+Freeciv.Web.Devel-3.3",
+                    "version_label": "-dev",
+                    "major_version": 2,
+                    "minor_version": 5,
+                    "patch_version": 99,
+                    "port": self.client_port,
+                    "password": None,
+                    "subject": None,
+                }
+                debug_event(f"sending login username={self.user_name} port={self.client_port}")
+                self.ws_client.send(login_message)
+
+            def handle_chat_msg(self, packet):
+                from freecivbot.utils.fc_events import E_UNDEFINED
+
+                message = packet["message"]
+                conn_id = packet["conn_id"]
+                event = packet["event"]
+
+                if message is None:
+                    return
+                if event is None or event < 0 or event >= E_UNDEFINED:
+                    print("Undefined message event type")
+                    print(packet)
+                    print("\r\n")
+                    packet["event"] = event = E_UNDEFINED
+
+                if conn_id in self.clstate.connections:
+                    message = "<b>" + self.clstate.connections[conn_id]["username"] + ":</b>" + message
+                else:
+                    if "/metamessage" in message:
+                        return
+                    if "Metaserver message string" in message:
+                        return
+
+                packet["message"] = message
+                debug_event(f"chat message: {message}")
+                print(packet)
+                print("\r\n")
+
+                if "You are logged in as" in message:
+                    debug_event("logged in; sending /set minplayers 1 and prepare_game")
+                    self.ws_client.send_message("/set minplayers 1")
+                    self.prepare_game()
+
+            def handle_conn_info(self, packet):
+                from freecivbot.connectivity.client_state import C_S_PREPARING
+                from freecivbot.utils.freecivlog import freelog
+
+                pconn = self.clstate.find_conn_by_id(packet["id"])
+
+                if not packet["used"]:
+                    if pconn is None:
+                        freelog(f"Server removed unknown connection {packet['id']}")
+                        return
+                    self.clstate.client_remove_cli_conn(pconn)
+                    pconn = None
+                else:
+                    pplayer = self.player_ctrl.valid_player_by_number(packet["player_num"])
+                    if pplayer is None:
+                        return
+                    packet["playing"] = pplayer
+
+                    if self.clstate.has_id(packet["id"]):
+                        self.clstate.init_state(packet)
+
+                    self.clstate.conn_list_append(packet)
+
+                if self.clstate.has_id(packet["id"]) and self.clstate.cur_player() != packet["playing"]:
+                    self.clstate.set_client_state(C_S_PREPARING)
+
+        self.impl = ConfiguredCivClientImpl(
+            bot,
+            user_name,
+            client_port=client_port,
+            visual_monitor=visual_monitor,
+        )
+
+
+class _ConfiguredCivConnection:
+    def __init__(self, civ_client, base_url: str, *, owner: "LiveFreecivSession", wait_for_server: int = 120, retry_interval: int = 5):
+        from math import ceil
+
+        import websocket
+
+        self._websocket = websocket
+        self.client = civ_client
+        self.base_url = base_url
+        self._owner = owner
+        self._loop = None
+        self._owner._connection = self
+        self.civserverport = self._reserve_client_port(base_url, civ_client.client_port)
+        self.client.client_port = self.civserverport
+        self.proxyport = 1000 + self.civserverport
+        debug_event(f"reserved civ port={self.civserverport} proxyport={self.proxyport}")
+        self._retry_interval = retry_interval
+        self._num_retries = int(ceil(wait_for_server / retry_interval))
+        self._cur_retry = 0
+        self._ws_url = self._build_ws_url(base_url)
+        self.network_init()
+
+    def _build_ws_url(self, base_url: str) -> str:
+        parsed = urlparse(base_url)
+        scheme = "wss" if parsed.scheme == "https" else "ws"
+        host = parsed.hostname or "localhost"
+        return f"{scheme}://{host}:{self.proxyport}/civsocket/{self.proxyport}"
+
+    def _reserve_client_port(self, base_url: str, requested_port: int) -> int:
+        parsed = urlparse(base_url)
+        scheme = parsed.scheme or "http"
+        host = parsed.hostname or "localhost"
+        port = parsed.port
+        if port is None:
+            port = 443 if scheme == "https" else 80
+        query = urlencode({"civserverport": requested_port})
+        launcher_url = f"{scheme}://{host}:{port}/civclientlauncher?{query}"
+        origin = f"{scheme}://{host}:{port}"
+        request = Request(
+            launcher_url,
+            method="POST",
+            headers={
+                "User-Agent": "Mozilla/5.0",
+                "Origin": origin,
+                "Referer": origin + "/",
+            },
+        )
+        with urlopen(request, timeout=10) as response:
+            result = response.headers.get("result")
+            reserved_port = response.headers.get("port")
+            debug_event(
+                f"launcher response status={response.status} result={result} port={reserved_port} body={response.read(200).decode('utf-8', 'ignore')}"
+            )
+        if result != "success" or reserved_port is None:
+            raise RuntimeError(f"failed to reserve freeciv client port via {launcher_url}")
+        return int(reserved_port)
+
+    def _retry(self):
+        self._cur_retry += 1
+        time.sleep(self._retry_interval)
+        return self._detect_server_up()
+
+    def _detect_server_up(self):
+        ws = self._websocket.WebSocket()
+        try:
+            debug_event(f"probing websocket {self._ws_url}")
+            ws.connect(self._ws_url, timeout=10)
+            debug_event("websocket probe succeeded")
+            return True
+        except Exception as err:
+            debug_event(f"websocket probe failed: {err!r}")
+            print("Connect not successful:", err, " retrying in %s seconds." % self._retry_interval)
+            if self._cur_retry < self._num_retries:
+                return self._retry()
+            return False
+        finally:
+            try:
+                ws.close()
+            except Exception:
+                pass
+
+    def network_init(self):
+        self._cur_retry = 0
+        print("Connecting to server at %s ..." % self.base_url)
+        if self._detect_server_up():
+            self.websocket_init()
+        else:
+            print("Connection could not be established!")
+
+    def websocket_init(self):
+        from tornado import ioloop
+
+        from freecivbot.connectivity.clinet import CivWSClient
+
+        asyncio.set_event_loop(asyncio.new_event_loop())
+        ioloop.IOLoop.clear_current()
+        self._loop = ioloop.IOLoop.current()
+
+        debug_event(f"starting tornado websocket client for {self._ws_url}")
+        client = CivWSClient(self.client)
+
+        def send_json(data):
+            if not client._ws_connection:
+                raise RuntimeError("Web socket connection is closed.")
+            msg = json.dumps(data, separators=(",", ":"))
+            client._ws_connection.write_message(msg)
+
+        client.send = send_json
+        client.connect(self._ws_url)
+
+        try:
+            self._loop.start()
+        except KeyboardInterrupt:
+            client.close()
+
+    def submit(self, fn) -> None:
+        if self._loop is None:
+            raise RuntimeError("freeciv connection loop is not ready")
+        done = threading.Event()
+        error: BaseException | None = None
+
+        def run():
+            nonlocal error
+            try:
+                fn()
+            except BaseException as exc:
+                error = exc
+            finally:
+                done.set()
+
+        self._loop.add_callback(run)
+        if not done.wait(timeout=10):
+            raise TimeoutError("timed out dispatching action to freeciv loop")
+        if error is not None:
+            raise error
+
+    def close(self) -> None:
+        if self._loop is None:
+            return
+        self.submit(self.client.close)
+
+
+class LiveFreecivSession:
+    def __init__(
+        self,
+        *,
+        username: str = "openenvbot",
+        client_port: int = 6000,
+        base_url: str = "http://localhost",
+        turn_timeout_s: float = 60.0,
+    ):
+        self.username = username
+        self.client_port = client_port
+        self.base_url = base_url
+        self.turn_timeout_s = turn_timeout_s
+
+        self._bot_wrapper: _InteractiveBot | None = None
+        self._client = None
+        self._connection: _ConfiguredCivConnection | None = None
+        self._thread: threading.Thread | None = None
+        self._ready = threading.Event()
+        self._snapshot_lock = threading.Lock()
+        self._snapshot: RawSnapshot | None = None
+        self._thread_error: BaseException | None = None
+        self._reset_counter = 0
+        self._session_seed = time.monotonic_ns() % 1_000_000
+
+    def reset(self, seed: int | None = None) -> RawSnapshot:
+        del seed
+        self.close()
+        self._reset_counter += 1
+        username = self._next_username()
+        client_port = self.client_port + ((self._session_seed + self._reset_counter - 1) % 3)
+
+        self._ready.clear()
+        self._thread_error = None
+        self._snapshot = None
+
+        self._bot_wrapper = _InteractiveBot(self)
+        self._client = _ConfiguredCivClient(
+            self._bot_wrapper.impl,
+            username,
+            client_port=client_port,
+            visual_monitor=False,
+        ).impl
+
+        def run() -> None:
+            try:
+                debug_event(f"session thread starting username={username} base_url={self.base_url} client_port={client_port}")
+                _ConfiguredCivConnection(self._client, self.base_url, owner=self)
+            except BaseException as exc:  # pragma: no cover - surfaced in waiters
+                debug_event(f"session thread error: {exc!r}")
+                self._thread_error = exc
+                self._ready.set()
+
+        self._thread = threading.Thread(target=run, name="freeciv-live-session", daemon=True)
+        self._thread.start()
+        return self._wait_for_snapshot("reset")
+
+    def apply_action(self, action_ref: ActionRef) -> RawSnapshot:
+        snapshot = self._require_snapshot()
+        action_list = snapshot.actions[action_ref.controller]
+        valid_actions = action_list.get_actions(action_ref.actor_id, valid_only=True)
+        action = None if valid_actions is None else valid_actions.get(action_ref.raw_action_key)
+        if action is None:
+            raise ValueError(
+                f"action {action_ref.raw_action_key} is no longer valid for {action_ref.controller}:{action_ref.actor_id}"
+            )
+        self._ready.clear()
+        connection = self._require_connection()
+        connection.submit(lambda: action_list.trigger_validated_action(action))
+        return self._wait_for_snapshot(action_ref.raw_action_key)
+
+    def end_turn(self) -> RawSnapshot:
+        if self._bot_wrapper is None:
+            raise RuntimeError("session has not been reset")
+        self._ready.clear()
+        connection = self._require_connection()
+        connection.submit(self._bot_wrapper.impl.end_turn)
+        return self._wait_for_snapshot("end_turn")
+
+    def close(self) -> None:
+        debug_event("closing live session")
+        if self._connection is not None:
+            try:
+                self._connection.close()
+            except Exception:
+                pass
+        elif self._client is not None:
+            try:
+                self._client.close()
+            except Exception:
+                pass
+        if self._thread is not None and self._thread.is_alive():
+            self._thread.join(timeout=5)
+        self._bot_wrapper = None
+        self._client = None
+        self._connection = None
+        self._thread = None
+        self._snapshot = None
+        self._thread_error = None
+        self._ready.clear()
+
+    def _publish_snapshot(self, snapshot: RawSnapshot) -> None:
+        debug_event(f"snapshot published turn={snapshot.turn}")
+        with self._snapshot_lock:
+            self._snapshot = snapshot
+        self._ready.set()
+
+    def _next_username(self) -> str:
+        suffix = str(self._session_seed + self._reset_counter)
+        prefix_len = max(1, 31 - len(suffix))
+        return f"{self.username[:prefix_len]}{suffix}"
+
+    def _require_connection(self) -> _ConfiguredCivConnection:
+        if self._connection is None:
+            raise RuntimeError("freeciv connection is not ready")
+        return self._connection
+
+    def _require_snapshot(self) -> RawSnapshot:
+        with self._snapshot_lock:
+            if self._snapshot is None:
+                raise RuntimeError("no live snapshot is available")
+            return self._snapshot
+
+    def _wait_for_snapshot(self, reason: str) -> RawSnapshot:
+        deadline = time.monotonic() + self.turn_timeout_s
+        debug_event(f"waiting for snapshot reason={reason} timeout={self.turn_timeout_s}")
+        while time.monotonic() < deadline:
+            if self._thread_error is not None:
+                raise RuntimeError(f"freeciv session failed during {reason}") from self._thread_error
+            if self._ready.wait(timeout=0.1):
+                if self._thread_error is not None:
+                    raise RuntimeError(f"freeciv session failed during {reason}") from self._thread_error
+                snapshot = self._require_snapshot()
+                if snapshot is not None:
+                    return snapshot
+        debug_event(f"snapshot wait timed out reason={reason}")
+        raise TimeoutError(f"timed out waiting for freeciv snapshot during {reason}")

freeciv_env/server/Dockerfile

@@ -0,0 +1,31 @@
+FROM omkarasoftware/freeciv-web:latest
+
+USER root
+RUN apt-get update && apt-get install -y \
+    git \
+    curl \
+    ca-certificates \
+ && rm -rf /var/lib/apt/lists/*
+RUN mkdir -p /app/env && chown -R docker:docker /app
+
+USER docker
+ENV HOME=/home/docker
+WORKDIR /app/env
+
+COPY --chown=docker:docker . /app/env
+RUN chmod +x /app/env/scripts/start_space.sh
+RUN curl -LsSf https://astral.sh/uv/install.sh | sh
+ENV PATH="/app/env/.venv/bin:/home/docker/.local/bin:$PATH"
+RUN uv python install 3.11
+RUN uv venv --python 3.11 /app/env/.venv
+RUN UV_PROJECT_ENVIRONMENT=/app/env/.venv uv sync --frozen --no-dev --no-editable
+
+ENV PYTHONPATH="/app/env:$PYTHONPATH"
+ENV ENABLE_WEB_INTERFACE=true
+ENV FREECIV_SERVER_URL=http://127.0.0.1
+ENV FREECIV_TURN_TIMEOUT_S=120
+
+HEALTHCHECK --interval=30s --timeout=5s --start-period=20s --retries=10 \
+    CMD curl -f http://localhost:8000/health || exit 1
+
+CMD ["/app/env/scripts/start_space.sh"]

freeciv_env/server/__init__.py

@@ -0,0 +1,3 @@
+from freeciv_env.server.freeciv_environment import FreecivEnvironment
+
+__all__ = ["FreecivEnvironment"]

freeciv_env/server/app.py

@@ -0,0 +1,144 @@
+from __future__ import annotations
+
+import glob
+import os
+from urllib.request import Request, urlopen
+
+from fastapi import Query
+from openenv.core.env_server import create_app
+
+from freeciv_env.adapter import prepare_observation
+from freeciv_env.models import FreecivAction, FreecivObservation
+from freeciv_env.runtime import DEBUG_EVENTS, LiveFreecivSession
+from freeciv_env.server.freeciv_environment import FreecivEnvironment
+
+
+def create_live_session() -> LiveFreecivSession:
+    return LiveFreecivSession(
+        username=os.getenv("FREECIV_USERNAME", "openenvbot"),
+        client_port=int(os.getenv("FREECIV_CLIENT_PORT", "6000")),
+        base_url=os.getenv("FREECIV_SERVER_URL", "http://127.0.0.1"),
+        turn_timeout_s=float(os.getenv("FREECIV_TURN_TIMEOUT_S", "60")),
+    )
+
+
+def create_freeciv_app(*, session_factory=create_live_session, max_turns: int | None = None):
+    if max_turns is None:
+        max_turns = int(os.getenv("FREECIV_MAX_TURNS", "50"))
+    return create_app(
+        lambda: FreecivEnvironment(session_factory=session_factory, max_turns=max_turns),
+        FreecivAction,
+        FreecivObservation,
+        env_name="freeciv_env",
+    )
+
+
+app = create_freeciv_app()
+
+
+@app.get("/debug/internal-status")
+def debug_internal_status() -> dict:
+    checks = []
+    for name, method, url in [
+        ("nginx", "GET", "http://127.0.0.1/"),
+        ("publite2", "GET", "http://127.0.0.1/pubstatus"),
+        ("tomcat", "GET", "http://127.0.0.1:8080/freeciv-web/"),
+        ("proxy7000", "GET", "http://127.0.0.1/civsocket/7000/status"),
+        ("proxy7001", "GET", "http://127.0.0.1/civsocket/7001/status"),
+        ("proxy7002", "GET", "http://127.0.0.1/civsocket/7002/status"),
+        ("launcher", "POST", "http://127.0.0.1/civclientlauncher?civserverport=6000"),
+    ]:
+        try:
+            request = Request(url, method=method)
+            with urlopen(request, timeout=10) as response:
+                body = response.read(200).decode("utf-8", "ignore")
+                checks.append(
+                    {
+                        "name": name,
+                        "ok": True,
+                        "status": response.status,
+                        "body": body,
+                    }
+                )
+        except Exception as exc:
+            checks.append({"name": name, "ok": False, "error": repr(exc)})
+    return {"checks": checks}
+
+
+@app.get("/debug/live-log")
+def debug_live_log() -> dict:
+    return {"events": list(DEBUG_EVENTS)}
+
+
+@app.get("/debug/freeciv-logs")
+def debug_freeciv_logs() -> dict:
+    logs = {}
+    for path in sorted(glob.glob("/docker/logs/*.log"))[-12:]:
+        try:
+            with open(path, "r", encoding="utf-8", errors="ignore") as handle:
+                lines = handle.readlines()[-80:]
+            logs[path] = "".join(lines)
+        except Exception as exc:
+            logs[path] = repr(exc)
+    return {"logs": logs}
+
+
+@app.get("/debug/startup-log")
+def debug_startup_log() -> dict:
+    path = "/tmp/start_space.log"
+    try:
+        with open(path, "r", encoding="utf-8", errors="ignore") as handle:
+            lines = handle.readlines()[-120:]
+        return {"path": path, "log": "".join(lines)}
+    except Exception as exc:
+        return {"path": path, "error": repr(exc)}
+
+
+@app.post("/debug/live-reset")
+def debug_live_reset(timeout_s: float = Query(default=120.0, ge=10.0, le=300.0)) -> dict:
+    session = create_live_session()
+    session.turn_timeout_s = timeout_s
+    try:
+        reset_snapshot = session.reset()
+        reset_observation = prepare_observation(
+            reset_snapshot,
+            reward=0.0,
+            done=False,
+            status="ready",
+            metadata={},
+        ).observation
+        next_snapshot = session.end_turn()
+        next_observation = prepare_observation(
+            next_snapshot,
+            reward=0.0,
+            done=False,
+            status="ok",
+            metadata={},
+        ).observation
+        return {
+            "ok": True,
+            "reset": {
+                "turn": reset_observation.turn,
+                "legal_actions": len(reset_observation.legal_actions),
+                "summary": reset_observation.summary,
+            },
+            "step": {
+                "turn": next_observation.turn,
+                "legal_actions": len(next_observation.legal_actions),
+                "summary": next_observation.summary,
+            },
+        }
+    except Exception as exc:
+        return {"ok": False, "error": repr(exc)}
+    finally:
+        session.close()
+
+
+def main() -> None:
+    import uvicorn
+
+    uvicorn.run(app, host="0.0.0.0", port=8000, ws_ping_interval=300, ws_ping_timeout=300)
+
+
+if __name__ == "__main__":
+    main()

freeciv_env/server/freeciv_environment.py

@@ -0,0 +1,163 @@
+from __future__ import annotations
+
+from typing import Callable
+from uuid import uuid4
+
+from openenv.core.env_server.interfaces import Environment
+
+from freeciv_env.adapter import (
+    ActionLookupKey,
+    ActionRef,
+    PreparedObservation,
+    RawSnapshot,
+    SnapshotMetrics,
+    action_lookup_key,
+    prepare_observation,
+)
+from freeciv_env.models import FreecivAction, FreecivObservation, FreecivState
+from freeciv_env.runtime import FreecivSession
+
+
+class FreecivEnvironment(Environment[FreecivAction, FreecivObservation, FreecivState]):
+    SUPPORTS_CONCURRENT_SESSIONS = False
+
+    def __init__(self, session_factory: Callable[[], FreecivSession], max_turns: int = 50):
+        super().__init__()
+        self._session_factory = session_factory
+        self.max_turns = max_turns
+        self._session: FreecivSession | None = None
+        self._snapshot: RawSnapshot | None = None
+        self._metrics: SnapshotMetrics | None = None
+        self._action_refs: dict[ActionLookupKey, ActionRef] = {}
+        self._state = FreecivState(episode_id=str(uuid4()), step_count=0)
+
+    def reset(
+        self,
+        seed: int | None = None,
+        episode_id: str | None = None,
+        **kwargs,
+    ) -> FreecivObservation:
+        del kwargs
+        self.close()
+        self._session = self._session_factory()
+        snapshot = self._session.reset(seed=seed)
+        prepared = prepare_observation(
+            snapshot,
+            reward=0.0,
+            done=self._is_done(snapshot),
+            status="ready",
+            metadata={},
+        )
+        self._commit(snapshot, prepared, episode_id=episode_id or str(uuid4()))
+        return prepared.observation
+
+    def step(
+        self,
+        action: FreecivAction,
+        timeout_s: float | None = None,
+        **kwargs,
+    ) -> FreecivObservation:
+        del timeout_s, kwargs
+        if self._session is None or self._snapshot is None or self._metrics is None:
+            raise RuntimeError("environment must be reset before step")
+
+        self._state.step_count += 1
+        if action.action_type == "end_turn":
+            next_snapshot = self._session.end_turn()
+            reward = self._reward_for_transition(action, self._metrics, next_snapshot)
+            prepared = prepare_observation(
+                next_snapshot,
+                reward=reward,
+                done=self._is_done(next_snapshot),
+                status="ok",
+                metadata={},
+            )
+            self._commit(next_snapshot, prepared, episode_id=self._state.episode_id)
+            return prepared.observation
+
+        ref = self._action_refs.get(action_lookup_key(action))
+        if ref is None:
+            prepared = prepare_observation(
+                self._snapshot,
+                reward=-0.25,
+                done=self._is_done(self._snapshot),
+                status="invalid_action",
+                metadata={"error": "action is not currently legal"},
+            )
+            self._commit(self._snapshot, prepared, episode_id=self._state.episode_id, replace_snapshot=False)
+            return prepared.observation
+
+        next_snapshot = self._session.apply_action(ref)
+        reward = self._reward_for_transition(action, self._metrics, next_snapshot)
+        prepared = prepare_observation(
+            next_snapshot,
+            reward=reward,
+            done=self._is_done(next_snapshot),
+            status="ok",
+            metadata={},
+        )
+        self._commit(next_snapshot, prepared, episode_id=self._state.episode_id)
+        return prepared.observation
+
+    @property
+    def state(self) -> FreecivState:
+        return self._state
+
+    def close(self) -> None:
+        if self._session is not None:
+            self._session.close()
+        self._session = None
+        self._snapshot = None
+        self._metrics = None
+        self._action_refs = {}
+
+    def _commit(
+        self,
+        snapshot: RawSnapshot,
+        prepared: PreparedObservation,
+        *,
+        episode_id: str,
+        replace_snapshot: bool = True,
+    ) -> None:
+        if replace_snapshot:
+            self._snapshot = snapshot
+        self._metrics = prepared.metrics
+        self._action_refs = prepared.action_refs
+        self._state = FreecivState(
+            episode_id=episode_id,
+            step_count=self._state.step_count,
+            turn=prepared.observation.turn,
+            score=prepared.observation.score,
+            known_tiles=prepared.observation.known_tiles,
+            visible_tiles=prepared.observation.visible_tiles,
+            city_count=prepared.observation.city_count,
+            unit_count=prepared.observation.unit_count,
+            techs_researched=prepared.observation.techs_researched,
+        )
+
+    def _reward_for_transition(
+        self,
+        action: FreecivAction,
+        previous: SnapshotMetrics,
+        next_snapshot: RawSnapshot,
+    ) -> float:
+        from freeciv_env.adapter import extract_metrics
+
+        current = extract_metrics(next_snapshot)
+        reward = {
+            "end_turn": 0.0,
+            "move_unit": 0.01,
+            "build_city": 0.10,
+            "set_city_production": 0.05,
+            "set_research": 0.05,
+        }[action.action_type]
+        reward += max(current.score - previous.score, 0.0) * 0.02
+        reward += max(current.known_tiles - previous.known_tiles, 0) * 0.01
+        reward += max(current.city_count - previous.city_count, 0) * 0.50
+        reward += max(current.techs_researched - previous.techs_researched, 0) * 0.25
+        return float(reward)
+
+    def _is_done(self, snapshot: RawSnapshot) -> bool:
+        player = snapshot.state.get("player", {})
+        alive = bool(player.get("my_is_alive", True))
+        return (not alive) or snapshot.turn >= self.max_turns

hackathon.md

@@ -0,0 +1,271 @@
+## **OpenEnv Hackathon Participant Guide**
+
+Welcome to the [OpenEnv Hackathon](https://cerebralvalley.ai/e/open-env-hackathon), hacker! 👋 We’re thrilled to have you on board.
+
+This guide is your all-in-one resource for the event, including schedule, rules, technical resources, problem statements, judging information, and more. Please read this carefully; most answers can be found here.
+
+## **1. Join the [PyTorch Discord Server](https://discord.gg/VBcf6VtfY6)**
+
+- You’ll be given a Hackathon Participant role by an admin, which will give you access to the hackathon-specific channels.
+
+- Here, you’ll be able to interact with hackers and sponsors, introduce yourselves, and form teams (for a maximum team size of **3**).
+
+- If you don't receive your role within **24 hours of joining,** please ping @CV.
+
+- Please submit your Discord username below so we can grant you the role
+
+[linkEmbed]
+
+## **2. Location**
+
+**|** Shack15 (1 Ferry Building, Suite 201, San Francisco CA. 94111)
+
+- **Venue Access:** Shack15 is on the 2nd floor of the Ferry Building. Go up the Ferry Building elevator to the second floor, and turn left. Here you will see the main entrance to Shack15. 
+
+- **Parking:** Parking near the Ferry Building is extremely limited. Consider parking farther out and taking Uber, Lyft, or Public Transportation. 
+
+[youtube]
+
+## **3. WiFi Information**
+
+- **Username:** SHACK15_Members
+
+- **Password:** M3mb3r$4L!f3
+
+## **4. Hackathon Schedule**
+
+**Saturday, March 7 (Outline)**
+
+- **9:00 AM:** Doors Open •󠁏 Breakfast Served •󠁏 Team Formation
+
+- **10:00 AM – 11:30AM**: Kick-off presentations with Meta, Hugging Face, UC Berkeley, CoreWeave, OpenPipe, Unsloth AI, Fleet AI, Mercor, Scaler AI Labs, Snorkel AI, Patronus AI, Halluminate and Scale AI
+
+- **11:30 AM:** Hacking Begins
+
+- **1:00 PM:** Lunch Served
+
+- **6:00 PM:** Dinner Served
+
+- **10:00 PM:** Doors Close •󠁏 Re-entry not permitted
+
+**Sunday, March 8 (Outline)**
+
+- **9:00AM:** Doors Open •󠁏 Breakfast Served
+
+- **1:00PM:** Hacking stops •󠁏 Submissions Due
+
+- **1:15PM:** First Round Judging Begins
+
+- **2:00PM:** Lunch Served
+
+- **3:00PM:** Final Round Judging Begins
+
+- **4:00PM:** Winners Announced and Closing
+
+- **5:00PM:** Doors Close
+
+All presentation slides can be found here
+
+[linkEmbed]
+
+## **5. Hackathon and Submission Rules**
+
+To keep things fair and aligned with our goals, all teams must follow these rules:
+
+- **Open Source:** Please ensure your repository is public.
+
+- **New Work Only:** All projects must be started from scratch during the hackathon with no previous work.
+
+- **Team Size:** Teams may have up to **3** members.
+
+- **Banned Projects:** Projects will be disqualified if they: violate legal, ethical, or platform policies, use code, data, or assets you do not have the rights to.
+
+- Your project **must** use OpenEnv (stable release 0.2.1) deployed on HF spaces
+
+- You must show a minimal training script for your environment using Unsloth or HF TRL in Colab.
+
+- You must upload a **one minute** demo video to YouTube talking about your submission.
+
+## **6. Hackathon Problem Statements**
+
+Your project must address at least **one of the five** required problem statements.
+
+- Some problem statements include **optional partner-sponsored sub-problem statements**, which are additional focus areas related to the main theme.
+
+- Your project may align with **multiple partner sub-problem statements**, but you can only be **judged for a maximum of two**. Please **select up to two** when submitting.
+
+- Projects that match these partner sub-problem statements are eligible for **extra partner prizes**, judged separately from the main track winners.
+
+- Each partner sub-problem statement carries a prize of **$10,000 USD**.
+
+**Statement 1: Multi-Agent Interactions**
+
+Environments for this theme involve cooperation, competition, negotiation, and coalition formation. Learning from these environments will enable agents to model the beliefs and incentives of others in partially observable settings. This drives theory-of-mind reasoning and emergent strategic behavior.
+
+- **Expected Outcome:** an environment that can be used to train multi-agent task handling in a LLM
+
+- **Example Environments:** Market simulations, compute-allocation negotiations, collaborative puzzle worlds, mixed cooperative/competitive strategy games.
+
+- **Partner Sub-Themes:**
+
+  - **Fleet AI:** Scalable Oversight: Environments that train oversight agents to monitor, analyze, and explain the behavior of other AI agents operating in complex, multi-agent settings.
+  - **Halluminate:** Multi-Actor Environments: Build a realistic environment where an agent interacts with and manages multiple actors (agents) to discover and achieve the task
+
+**Statement 2: (Super) Long-Horizon Planning & Instruction Following**
+
+You will build environments that require deep, multi-step reasoning with sparse or delayed rewards. After using these environments, the goal is to enable agents to decompose goals, track state over extended trajectories, and recover from early mistakes. The aim is to push beyond shallow next-token reasoning toward structured planning and durable internal representations. 
+
+- **Expected Outcome:** an environment that can capture and improve LLM behaviour on challenging long horizon tasks that need long running sessions beyond context memory limits. 
+
+- **Example Environments:** Research-planning simulators, large-scale codebase refactoring tasks, strategic resource management worlds, long-horizon logistics optimization, extremely complicated long-horizon instruction following (e.g., 300 instructions scattered around).
+
+- **Partner Sub-Themes:**
+
+  - **Mercor:** Make an environment with capped/uncapped rewards where frontier model rewards scale with token output.
+
+  - **Scale AI:** Environments for long horizon workflows for non-code use cases within a business setting: focusing on either Sales, Project management, or HR & IT.
+
+**Statement 3: World Modeling**
+
+- **Statement 3.1: Professional Tasks:** Here you will develop environments that require real interaction with tools, APIs, or dynamic systems where the model is expected to do real hard work instead of exploiting short-cuts to arrive at the desired outcome. Learning from these environments will enable agents to maintain consistent internal state, update beliefs based on outcomes, and orchestrate multi-step workflows. The goal is to strengthen causal reasoning and persistent world models.
+
+  - **Expected Outcome:** an environment capturing nuances of a defined partially observable world and improve LLM interaction with it
+
+  - **Example Environments:** Dynamic browser/API ecosystems, enterprise applications, scientific workflow loops (papers → code → experiments), economic simulations with feedback, tool-discovery benchmarks.
+
+  - **Partner Sub-Theme:**
+
+    - **Scaler AI Labs:** Multi-App RL Environment for Enterprise Workflows: Create RL environments to demonstrate complex workflows, business rule nuances etc in a large enterprise
+
+- **Statement 3.2: Personalized Tasks:** Here we will develop an environment that offers real personalized task handling, imagine replying to personal messages or handling dinner conflicts due to work conflicts, replying to tough emails. Think any personal assistant tasks.
+
+  - **Expected Outcome:** An environment that gives the model a realistic simulation of handling personal tasks, conflicts and managing them as delegations
+
+  - **Example Environments:** Executive Assistant Meeting Planner, Dinner and drive planning, email and message replying, etc
+
+  - **Partner Sub-Theme:**
+
+    - **Patronus AI:** Consumer Workflows with Schema Drift: Multi-step consumer workflow environments where the underlying data schemas, API contracts, and t&cs/policies/rules change.
+
+**Statement 4: Self-Improvement**
+
+The focus here is to create environments where agents can learn to generate new challenges, escalate difficulty, and improve through self-play or adaptive curricula. Rather than optimizing fixed tasks, the goal is for agents to learn to drive their own capability growth. The objective is recursive skill amplification.
+
+- **Expected Outcome:** an environment for improving self-play of a LLM over a defined set of tasks
+
+- **Example Environments:** Self-play negotiation arenas, auto-generated math/proof tasks, evolving coding competitions, adaptive RL curricula.
+
+- **Partner Sub-Theme:**
+
+  - **Snorkel AI:** Simulated Experts-in-the-Loop: Environment that simulates interactions with real subject-matter experts, with changing requirements / preferences.
+
+**Statement 5: Wild Card - Impress Us!**
+
+We do not want to limit your focus if your idea doesn’t fit the boxes above, we want and WILL reward out of box tasks, please be creative but remember to add submissions that meaningfully add value to LLM training on a certain task.
+
+More details about each theme can be found here:
+
+[linkEmbed]
+
+## **7. CV Hackathon Winners**
+
+[linkEmbed]
+
+## **8. OpenEnv Provided Resources**
+
+**Please read through the entire slideshow here. This includes:**
+
+- OpenEnv Fundamentals, Architecture
+- Local Dev, Docker, and HF Spaces Deployment
+- OpenEnv in Practice
+- Training (TRL & Unsloth)
+- How-to-Access-Infrastructure (including GPU Request Form)
+
+[linkEmbed]
+
+## **9. Partner Provided Resources**
+
+- **Unsloth AI Resources**
+  - RL notebooks: <https://unsloth.ai/docs/get-started/unsloth-notebooks#grpo-reasoning-rl>
+  - All notebooks: <https://unsloth.ai/docs/get-started/unsloth-notebooks>
+  - GitHub notebook index: <https://github.com/unslothai/notebooks/tree/main/nb>
+  - H100 / OpenEnv recommendation: use the BF16 gpt-oss 20B OpenEnv notebook for faster H100 runs: <https://colab.research.google.com/github/unslothai/notebooks/blob/main/nb/OpenEnv_gpt_oss_(20B)_Reinforcement_Learning_2048_Game_BF16.ipynb>
+  - For that notebook, reduce `max_steps` to `300` to make the process faster.
+  - If GRPO is too slow, prefer smaller-model notebooks with `fast_inference = True`, for example:
+    - <https://colab.research.google.com/github/unslothai/notebooks/blob/main/nb/Qwen3_(4B)-GRPO.ipynb>
+    - <https://colab.research.google.com/github/unslothai/notebooks/blob/main/nb/Advanced_Llama3_2_(3B)_GRPO_LoRA.ipynb>
+  - You will need to edit those notebooks to include OpenEnv calls.
+  - If vLLM GRPO runs fail, try using a fresh virtualenv:
+
+    ```bash
+    python -m venv unsloth_env
+    source unsloth_env/bin/activate
+    pip install --upgrade pip && pip install uv
+    uv pip install unsloth vllm --torch-backend=auto
+    ```
+
+  - If Unsloth is already installed, update it for the latest GRPO bugfixes:
+
+    ```bash
+    pip install --upgrade --no-cache-dir --no-deps unsloth unsloth_zoo
+    ```
+
+- **Mercor Resources**
+  - Dataset: <https://huggingface.co/datasets/mercor/apex-agents>
+  - Archipelago repo to run the eval: <https://github.com/Mercor-Intelligence/archipelago>
+  - APEX-Agents paper: <https://arxiv.org/abs/2601.14242>
+- **Hugging Face Resources**
+  - **$30** in Compute and Inference Credits
+  - To claim your credits, set up a HF account here: <https://huggingface.co/join>
+  - Then, follow this link: <https://huggingface.co/openenv-community>
+  - You will be granted **$30** of compute and inference credits!
+- **Northflank Resources**
+  - Each team gets an H100
+  - Northflank instructions
+
+    [linkEmbed]
+  - Join the NorthFlank discord channel for any questions 
+  - Please fill out this form:
+
+    [linkEmbed]
+
+
+- **Cursor Resources**
+  - **$50** in Cursor Credits, **apply below**
+
+    [linkEmbed]
+
+## **10. Judging & Submissions**
+
+Judges will be taking place on **Sunday, March 8**. These judges are evaluating your **technical demos** in the following categories. *Show us what you have built* to solve our problem statements. Please **do not** show us a presentation. We'll be checking to ensure your project was built **entirely during the event**; no previous work is allowed. 
+
+**|** **Teams should submit [here](https://cerebralvalley.ai/e/openenv-hackathon-sf/hackathon/submit) when they have completed hacking.** In the submission form, you will have to upload a **one minute** demo video on YouTube talking about your submission. You must also show a minimal training script for your environment using Unsloth or HF TRL in Colab.
+
+**Please ensure your project uses** use OpenEnv (stable release 0.2.1) deployed on HF spaces.
+
+[linkEmbed]
+
+**Judging Criteria**
+
+- **Environment Innovation (40%) -** Is the environment novel, creative, or challenging? Does it meaningfully test the agent’s behavior?
+- **Storytelling (30%) -** Does the team clearly explain the problem, environment, and agent behavior? Is the demo engaging and easy to follow?
+- **Training Script Showing Improvement in Rewards (20%) -** Does the demo provide observable evidence of training progress (reward curves, metrics, or before/after behavior)? 
+- **Reward and Training Pipeline Setup (10%) -** Is the reward logic coherent, and does the pipeline produce meaningful improvement in the agent’s inference (how it acts in the environment)?
+
+**Judging Process**
+
+**|** Judging proceeds in two rounds:
+
+- Hackers will be assigned groups of judges; \~3 minutes to pitch followed by 1-2 minutes of Q/A
+
+- The top **six** teams in ranking will get to demo on stage to a panel of judges; \~3 minutes to pitch followed by 2-3 minutes for Q/A.
+
+## **11. Prizes**
+
+- **1st Place:** $15,000 USD Cash
+
+- **2nd Place:** $9,000 USD Cash
+
+- **3rd Place:** $6,000 USD Cash
+
+## **❓If you have any questions, please email [wania@cerebralvalley.ai](mailto:wania@cerebralvalley.ai) or message on Discord.**

models.py

@@ -0,0 +1 @@
+from freeciv_env.models import *

notes.md

@@ -0,0 +1,3 @@
+multi-agent (track 1) - freeciv
+long context (track 2) - freeciv
+self improving? (track 4) - mechinterp

openenv.yaml

@@ -0,0 +1,4 @@
+name: freeciv_env
+description: OpenEnv wrapper around freeciv-bot for long-horizon strategy play.
+version: 0.1.0
+entrypoint: freeciv_env.server.app:app

outline.md

@@ -0,0 +1,178 @@
+# Demo outline
+
+## Open these tabs first
+
+Local resources:
+- `pres/index.html`
+- `pres/training_results.html`
+- `pres/trajectory.html`
+- `pres/training_script.html`
+- `pres/reward_curve.png`
+- `pres/before_after_reward.png`
+
+Remote resources:
+- HF Space repo: <https://huggingface.co/spaces/thomasm6m6/freeciv_env>
+- HF Space app: <https://thomasm6m6-freeciv-env.hf.space>
+
+Supporting files:
+- reward data: `pres/reward_steps.csv`
+- training script: `scripts/train_grpo_fast.py`
+- env config: `openenv.yaml`
+
+## What we have ready
+
+- real OpenEnv environment for Freeciv
+- real live backend on H100 via Freeciv Web
+- successful GRPO training run on the live backend
+- reward curve PNG
+- before/after reward PNG
+- live trajectory page with real observations + legal actions
+- note: use reward improvement as the before/after story; raw checkpoint-to-checkpoint action examples were too noisy to be worth showing live
+- minimal training script page
+- HF Space deployed: `thomasm6m6/freeciv_env`
+
+## What not to spend time on
+
+- long architecture explanation
+- low-level websocket/runtime debugging
+- model internals
+- many charts
+
+Use the product demo + reward improvement as the center of the pitch.
+
+---
+
+## 1 minute YouTube flow
+
+### 0:00–0:10
+Open: `pres/trajectory.html`
+
+Say:
+- We built a real OpenEnv environment for Freeciv, a long-horizon strategy game.
+- The model sees text observations and legal actions, and acts turn by turn against a live backend.
+
+### 0:10–0:22
+Stay on `pres/trajectory.html`
+
+Say:
+- This is not a toy prompt task.
+- It has delayed reward, persistent world state, multiple units, city-building, and long-horizon planning.
+- That maps directly to the hackathon’s long-horizon planning and world-modeling tracks.
+
+### 0:22–0:38
+Switch to `pres/training_script.html`
+
+Say:
+- We also built the minimal RL training loop with Unsloth + TRL GRPO.
+- The script collects live Freeciv states, formats them into prompts, and trains a policy on the real environment.
+
+### 0:38–0:55
+Switch to `pres/training_results.html`
+
+Say:
+- We ran training on the H100 against the live Freeciv backend.
+- Reward improved from 0.125 at the start to 1.0 by the end of the run.
+- This gives observable training progress, which is the key hackathon requirement.
+
+### 0:55–1:00
+Optional final cut to HF Space repo URL
+
+Say:
+- The environment is packaged as OpenEnv and deployed to Hugging Face Spaces for submission.
+
+---
+
+## 3 minute live pitch flow
+
+### 0:00–0:25 — problem
+Open: `pres/trajectory.html`
+
+Say:
+- We wanted a real LLM RL environment for long-horizon strategic planning.
+- Freeciv is a strong fit because it has persistent state, delayed reward, many legal actions, and requires planning across turns.
+
+### 0:25–1:05 — show the environment
+Stay on `pres/trajectory.html`
+
+Point out:
+- text-first observation
+- legal actions
+- units / cities / economy summaries
+- live backend on H100
+
+Say:
+- The agent does not get a canned benchmark prompt.
+- It interacts with a real running world and must choose from legal actions each turn.
+
+### 1:05–1:35 — show the training loop
+Open: `pres/training_script.html`
+
+Say:
+- This is the minimal GRPO loop.
+- We use live Freeciv sessions, prepare observations, build prompts, and train with Unsloth + TRL.
+- The important thing is that the training loop is small and actually runs on the real backend.
+
+### 1:35–2:25 — show training improvement
+Open: `pres/training_results.html`
+
+Say:
+- This is the core result.
+- Reward increases over training steps on real Freeciv states.
+- Start: 0.125. End: 1.0.
+- This is the evidence that the environment and reward pipeline are coherent enough to drive learning.
+
+If short on time, only show:
+- reward curve
+- before/after reward bars
+
+### 2:25–2:50 — why this matters
+Stay on `pres/training_results.html`
+
+Say:
+- This fits Statement 2: long-horizon planning.
+- It also fits Statement 3.1: world modeling, because the agent interacts with a real dynamic system and must maintain state over time.
+
+### 2:50–3:00 — close
+Open: HF Space repo URL or `pres/index.html`
+
+Say:
+- The environment is packaged in OpenEnv, runs with a real backend, has a minimal RL script, and already shows reward improvement.
+
+---
+
+## Likely Q/A answers
+
+### Why Freeciv?
+- It is long-horizon, strategic, partially observable, and naturally multi-step.
+- It is much closer to real planning than one-shot QA.
+
+### What exactly is the observation/action interface?
+- Observation is text-first: turn summary, economy, units, cities, map, legal actions.
+- Actions are structured: end turn, move unit, build city, set city production, set research.
+
+### Is the backend real?
+- Yes. Training was run against a live Freeciv Web backend on the H100.
+
+### What evidence do you have that training worked?
+- The reward curve in `pres/training_results.html`.
+- It rises from 0.125 to 1.0 during the live run.
+
+### Why not show a bigger model?
+- For the hackathon, reliability and observable reward improvement mattered more than model scale.
+- A smaller model let us get an end-to-end live run working on the real backend.
+
+### What is still incomplete?
+- The environment currently exposes a small action subset rather than the full Freeciv action surface.
+- The main accomplishment is that live interaction and RL training now work end to end.
+
+---
+
+## If something breaks during the pitch
+
+Fallback tab order:
+1. `pres/training_results.html`
+2. `pres/trajectory.html`
+3. `pres/training_script.html`
+4. HF Space repo URL
+
+If the live environment demo is flaky, just narrate from the trajectory page and go straight to the reward curve.

pres/index.html

@@ -0,0 +1,18 @@
+<!doctype html>
+<html><head><meta charset='utf-8'><title>Freeciv demo resources</title>
+<style>
+body { font-family: -apple-system, BlinkMacSystemFont, sans-serif; max-width: 900px; margin: 40px auto; line-height: 1.5; padding: 0 20px; }
+ul { line-height: 1.9; }
+code { background: #f6f8fa; padding: 2px 6px; border-radius: 6px; }
+</style></head><body>
+<h1>Freeciv OpenEnv demo resources</h1>
+<ul>
+  <li><a href='training_results.html'>Training results</a></li>
+  <li><a href='trajectory.html'>Live trajectory</a></li>
+  <li><a href='training_script.html'>Minimal training script</a></li>
+  <li><a href='reward_curve.png'>Reward curve PNG</a></li>
+  <li><a href='before_after_reward.png'>Before/after reward PNG</a></li>
+</ul>
+<p>HF Space: <code>https://huggingface.co/spaces/thomasm6m6/freeciv_env</code></p>
+<p>Space app domain: <code>https://thomasm6m6-freeciv-env.hf.space</code></p>
+</body></html>

pres/reward_steps.csv

@@ -0,0 +1,11 @@
+step,reward,reward_std
+1,0.125,0.25
+2,0.375,0.5386751294136047
+3,0.25,0.5
+4,0.5,0.5773502588272095
+5,0.625,0.5386751294136047
+6,0.875,0.25
+7,0.75,0.5
+8,0.875,0.25
+9,0.75,0.5
+10,1.0,0.0

pres/training_results.html

@@ -0,0 +1,27 @@
+<!doctype html>
+<html><head><meta charset='utf-8'><title>Training results</title>
+<style>
+body { font-family: -apple-system, BlinkMacSystemFont, sans-serif; max-width: 1000px; margin: 40px auto; line-height: 1.45; padding: 0 20px; }
+.card { background: #f6f8fa; border-radius: 10px; padding: 18px; margin: 16px 0; }
+img { max-width: 100%; border: 1px solid #ddd; border-radius: 8px; }
+table { border-collapse: collapse; width: 100%; }
+td, th { border-bottom: 1px solid #ddd; padding: 8px; text-align: left; }
+</style></head><body>
+<h1>Training results</h1>
+<div class='card'>
+<b>Live backend:</b> real Freeciv Web on H100<br>
+<b>Model:</b> Qwen/Qwen3.5-0.8B + Unsloth LoRA + TRL GRPO<br>
+<b>Run:</b> 10 steps, 32 live states, batch size 8<br>
+<b>Train runtime:</b> None
+</div>
+<div class='card'>
+<b>Observed reward improvement:</b> 0.125 → 1.000<br>
+<b>Best visible point:</b> step 10 reward 1.000
+</div>
+<h2>Reward curve</h2>
+<p><img src='reward_curve.png' alt='reward curve'></p>
+<h2>Start vs end</h2>
+<p><img src='before_after_reward.png' alt='before after reward'></p>
+<h2>Per-step reward</h2>
+<table><tr><th>step</th><th>reward</th><th>reward std</th></tr><tr><td>1</td><td>0.125</td><td>0.250</td></tr><tr><td>2</td><td>0.375</td><td>0.539</td></tr><tr><td>3</td><td>0.250</td><td>0.500</td></tr><tr><td>4</td><td>0.500</td><td>0.577</td></tr><tr><td>5</td><td>0.625</td><td>0.539</td></tr><tr><td>6</td><td>0.875</td><td>0.250</td></tr><tr><td>7</td><td>0.750</td><td>0.500</td></tr><tr><td>8</td><td>0.875</td><td>0.250</td></tr><tr><td>9</td><td>0.750</td><td>0.500</td></tr><tr><td>10</td><td>1.000</td><td>0.000</td></tr></table>
+</body></html>

pres/training_script.html

@@ -0,0 +1,150 @@
+<!doctype html>
+<html><head><meta charset='utf-8'><title>Minimal training script</title>
+<style>
+body { font-family: -apple-system, BlinkMacSystemFont, sans-serif; max-width: 1000px; margin: 40px auto; padding: 0 20px; }
+pre { background: #0d1117; color: #c9d1d9; padding: 16px; border-radius: 8px; overflow-x: auto; }
+code { font-family: ui-monospace, SFMono-Regular, Menlo, monospace; }
+</style></head><body>
+<h1>Minimal training script</h1>
+<p>Key file: <code>scripts/train_grpo_fast.py</code></p>
+<pre><code>from __future__ import annotations
+
+import argparse
+import os
+
+os.environ.setdefault(&quot;TOKENIZERS_PARALLELISM&quot;, &quot;false&quot;)
+os.environ.setdefault(&quot;UNSLOTH_RETURN_LOGITS&quot;, &quot;1&quot;)
+os.environ.setdefault(&quot;UNSLOTH_DISABLE_AUTO_UPDATES&quot;, &quot;1&quot;)
+
+from unsloth import FastLanguageModel
+from datasets import Dataset
+from trl import GRPOConfig, GRPOTrainer
+
+from freeciv_env.adapter import prepare_observation
+from freeciv_env.grpo import SYSTEM_PROMPT, build_turn_prompt, oracle_action_index, reward_from_oracle
+from freeciv_env.runtime import LiveFreecivSession
+
+
+def parse_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(&quot;--env-url&quot;, default=&quot;http://127.0.0.1&quot;)
+    parser.add_argument(&quot;--model-id&quot;, default=&quot;Qwen/Qwen3.5-0.8B&quot;)
+    parser.add_argument(&quot;--dataset-size&quot;, type=int, default=512)
+    parser.add_argument(&quot;--max-steps&quot;, type=int, default=50)
+    parser.add_argument(&quot;--batch-size&quot;, type=int, default=16)
+    parser.add_argument(&quot;--num-generations&quot;, type=int, default=4)
+    parser.add_argument(&quot;--episode-horizon&quot;, type=int, default=4)
+    parser.add_argument(&quot;--max-prompt-length&quot;, type=int, default=768)
+    parser.add_argument(&quot;--max-completion-length&quot;, type=int, default=8)
+    parser.add_argument(&quot;--learning-rate&quot;, type=float, default=5e-6)
+    parser.add_argument(&quot;--lora-rank&quot;, type=int, default=16)
+    parser.add_argument(&quot;--output-dir&quot;, default=&quot;outputs/qwen35_08b_grpo&quot;)
+    parser.add_argument(&quot;--save-steps&quot;, type=int, default=50)
+    return parser.parse_args()
+
+
+
+def collect_dataset(env_url: str, dataset_size: int, episode_horizon: int) -&gt; Dataset:
+    rows = {&quot;prompt&quot;: [], &quot;best_index&quot;: []}
+    while len(rows[&quot;prompt&quot;]) &lt; dataset_size:
+        session = LiveFreecivSession(base_url=env_url, turn_timeout_s=120)
+        try:
+            snapshot = session.reset()
+            for turn_index in range(episode_horizon):
+                observation = prepare_observation(
+                    snapshot,
+                    reward=0.0,
+                    done=False,
+                    status=&quot;running&quot;,
+                ).observation
+                best_index = oracle_action_index(observation.legal_actions)
+                rows[&quot;prompt&quot;].append(build_turn_prompt(observation))
+                rows[&quot;best_index&quot;].append(best_index)
+                if len(rows[&quot;prompt&quot;]) &gt;= dataset_size or turn_index + 1 &gt;= episode_horizon:
+                    break
+                snapshot = session.end_turn()
+        finally:
+            session.close()
+    return Dataset.from_dict(rows)
+
+
+
+def load_model(model_id: str, max_seq_length: int, lora_rank: int):
+    model, tokenizer = FastLanguageModel.from_pretrained(
+        model_name=model_id,
+        max_seq_length=max_seq_length,
+        load_in_4bit=False,
+        load_in_16bit=True,
+        full_finetuning=False,
+        fast_inference=False,
+    )
+    model = FastLanguageModel.get_peft_model(
+        model,
+        r=lora_rank,
+        target_modules=[
+            &quot;q_proj&quot;,
+            &quot;k_proj&quot;,
+            &quot;v_proj&quot;,
+            &quot;o_proj&quot;,
+            &quot;gate_proj&quot;,
+            &quot;up_proj&quot;,
+            &quot;down_proj&quot;,
+        ],
+        lora_alpha=lora_rank * 2,
+        lora_dropout=0,
+        bias=&quot;none&quot;,
+        use_gradient_checkpointing=False,
+        random_state=3407,
+        max_seq_length=max_seq_length,
+    )
+    return model, tokenizer
+
+
+
+def apply_chat_template(dataset: Dataset, tokenizer) -&gt; Dataset:
+    def format_row(row):
+        messages = [
+            {&quot;role&quot;: &quot;system&quot;, &quot;content&quot;: SYSTEM_PROMPT},
+            {&quot;role&quot;: &quot;user&quot;, &quot;content&quot;: row[&quot;prompt&quot;]},
+        ]
+        return {
+            &quot;prompt&quot;: tokenizer.apply_chat_template(
+                messages,
+                tokenize=False,
+                add_generation_prompt=True,
+                enable_thinking=False,
+            )
+        }
+
+    return dataset.map(format_row)
+
+
+
+def main() -&gt; None:
+    args = parse_args()
+    max_seq_length = args.max_prompt_length + args.max_completion_length
+    dataset = collect_dataset(args.env_url, args.dataset_size, args.episode_horizon)
+    model, tokenizer = load_model(args.model_id, max_seq_length, args.lora_rank)
+    dataset = apply_chat_template(dataset, tokenizer)
+
+    training_args = GRPOConfig(
+        learning_rate=args.learning_rate,
+        weight_decay=0.01,
+        warmup_ratio=0.05,
+        lr_scheduler_type=&quot;cosine&quot;,
+        optim=&quot;adamw_torch_fused&quot;,
+        logging_steps=1,
+        log_completions=False,
+        per_device_train_batch_size=args.batch_size,
+        gradient_accumulation_steps=1,
+        num_generations=args.num_generations,
+        max_prompt_length=args.max_prompt_length,
+        max_completion_length=args.max_completion_length,
+        max_steps=args.max_steps,
+        save_steps=args.save_steps,
+        max_grad_norm=0.3,
+        bf16=True,
+        report_to=&quot;none&quot;,
+        beta=0.0,
+        loss_type=&quot;dr_grpo&quot;,</code></pre>
+</body></html>

pres/trajectory.html

@@ -0,0 +1,62 @@
+<!doctype html>
+<html><head><meta charset='utf-8'><title>Freeciv live trajectory</title>
+<style>
+body { font-family: -apple-system, BlinkMacSystemFont, sans-serif; max-width: 1000px; margin: 40px auto; line-height: 1.4; padding: 0 20px; }
+pre { background: #f6f8fa; padding: 16px; border-radius: 8px; white-space: pre-wrap; }
+section { margin-bottom: 28px; }
+</style></head><body>
+<h1>Freeciv live trajectory</h1>
+<p>Real snapshots collected from the live Freeciv Web backend on the H100.</p>
+<section><h2>Turn 1 snapshot 1</h2><pre>Turn 1
+Score 0.0
+Map: 0 known tiles, 0 visible tiles
+Economy: 50 gold, science rate 60%
+Cities: 0
+Units: 5
+- Unit 102: Settlers, hp 20, moves_left 3, build_city=true, move_dirs=[0, 1, 2, 3, 4, 5, 6, 7]
+- Unit 110: Settlers, hp 20, moves_left 3, build_city=true, move_dirs=[0, 1, 2, 3, 4, 5, 6, 7]
+- Unit 111: Workers, hp 10, moves_left 3, build_city=false, move_dirs=[0, 1, 2, 3, 4, 5, 6, 7]
+- Unit 112: Workers, hp 10, moves_left 3, build_city=false, move_dirs=[0, 1, 2, 3, 4, 5, 6, 7]
+- Unit 113: Explorer, hp 10, moves_left 3, build_city=false, move_dirs=[0, 1, 2, 3, 4, 5, 6, 7]
+Techs researched: 1
+Legal actions exposed: 50</pre><h3>Legal actions (sample)</h3><pre>0. build_city — Build a city with unit 102
+1. build_city — Build a city with unit 110
+2. end_turn — End the current turn
+3. move_unit — Move unit 102 in direction 0
+4. move_unit — Move unit 102 in direction 1
+5. move_unit — Move unit 102 in direction 2</pre></section><section><h2>Turn 1 snapshot 2</h2><pre>Turn 1
+Score 0.0
+Map: 0 known tiles, 0 visible tiles
+Economy: 50 gold, science rate 60%
+Cities: 0
+Units: 5
+- Unit 102: Settlers, hp 20, moves_left 3, build_city=true, move_dirs=[0, 1, 2, 3, 4, 5, 6, 7]
+- Unit 110: Settlers, hp 20, moves_left 3, build_city=true, move_dirs=[0, 1, 2, 3, 4, 5, 6, 7]
+- Unit 111: Workers, hp 10, moves_left 3, build_city=false, move_dirs=[0, 1, 2, 3, 4, 5, 6, 7]
+- Unit 112: Workers, hp 10, moves_left 3, build_city=false, move_dirs=[0, 1, 2, 3, 4, 5, 6, 7]
+- Unit 113: Explorer, hp 10, moves_left 3, build_city=false, move_dirs=[0, 1, 2, 3, 4, 5, 6, 7]
+Techs researched: 1
+Legal actions exposed: 50</pre><h3>Legal actions (sample)</h3><pre>0. build_city — Build a city with unit 102
+1. build_city — Build a city with unit 110
+2. end_turn — End the current turn
+3. move_unit — Move unit 102 in direction 0
+4. move_unit — Move unit 102 in direction 1
+5. move_unit — Move unit 102 in direction 2</pre></section><section><h2>Turn 2 snapshot 3</h2><pre>Turn 2
+Score 0.0
+Map: 0 known tiles, 0 visible tiles
+Economy: 50 gold, science rate 60%
+Cities: 0
+Units: 5
+- Unit 102: Settlers, hp 20, moves_left 3, build_city=true, move_dirs=[0, 1, 2, 3, 4, 5, 6, 7]
+- Unit 110: Settlers, hp 20, moves_left 3, build_city=true, move_dirs=[0, 1, 2, 3, 4, 5, 6, 7]
+- Unit 111: Workers, hp 10, moves_left 3, build_city=false, move_dirs=[0, 1, 2, 3, 4, 5, 6, 7]
+- Unit 112: Workers, hp 10, moves_left 3, build_city=false, move_dirs=[0, 1, 2, 3, 4, 5, 6, 7]
+- Unit 113: Explorer, hp 10, moves_left 3, build_city=false, move_dirs=[0, 1, 2, 3, 4, 5, 6, 7]
+Techs researched: 1
+Legal actions exposed: 50</pre><h3>Legal actions (sample)</h3><pre>0. build_city — Build a city with unit 102
+1. build_city — Build a city with unit 110
+2. end_turn — End the current turn
+3. move_unit — Move unit 102 in direction 0
+4. move_unit — Move unit 102 in direction 1
+5. move_unit — Move unit 102 in direction 2</pre></section>
+</body></html>

pyproject.toml

@@ -0,0 +1,41 @@
+[project]
+name = "freeciv-env"
+version = "0.1.0"
+description = "OpenEnv environment for Freeciv via freeciv-bot"
+readme = "README.md"
+requires-python = ">=3.11"
+dependencies = [
+  "openenv-core[core]==0.2.1",
+  "freecivbot @ git+https://github.com/chris1869/freeciv-bot.git",
+  "uvicorn>=0.35.0",
+]
+
+[project.scripts]
+server = "server.app:main"
+
+[project.optional-dependencies]
+dev = [
+  "pytest>=8.4.1",
+  "requests>=2.32.5",
+]
+train = [
+  "accelerate>=1.10.0",
+  "bitsandbytes>=0.47.0",
+  "datasets>=4.0.0",
+  "trl>=0.24.0",
+  "unsloth>=2026.3.4",
+]
+
+[build-system]
+requires = ["setuptools>=80", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[tool.setuptools]
+packages = ["freeciv_env", "freeciv_env.server", "server"]
+
+[tool.pytest.ini_options]
+pythonpath = ["."]
+testpaths = ["tests"]
+markers = [
+  "integration: requires a live freeciv-web runtime",
+]

qwen35_live_long_trainer_state.json

@@ -0,0 +1,304 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.625,
+  "eval_steps": 500,
+  "global_step": 10,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 2.375,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 3.0,
+      "completions/max_terminated_length": 3.0,
+      "completions/mean_length": 2.375,
+      "completions/mean_terminated_length": 2.375,
+      "completions/min_length": 2.0,
+      "completions/min_terminated_length": 2.0,
+      "epoch": 0.0625,
+      "frac_reward_zero_std": 0.5,
+      "grad_norm": 5.300995349884033,
+      "kl": 0.0,
+      "learning_rate": 0.0,
+      "loss": 0.01562187448143959,
+      "num_tokens": 8343.0,
+      "reward": 0.125,
+      "reward_std": 0.25,
+      "rewards/reward_from_oracle/mean": 0.125,
+      "rewards/reward_from_oracle/std": 0.3535533845424652,
+      "step": 1
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 2.375,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 3.0,
+      "completions/max_terminated_length": 3.0,
+      "completions/mean_length": 2.375,
+      "completions/mean_terminated_length": 2.375,
+      "completions/min_length": 2.0,
+      "completions/min_terminated_length": 2.0,
+      "epoch": 0.125,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 9.095938682556152,
+      "kl": 0.0,
+      "learning_rate": 5e-06,
+      "loss": 0.029151180759072304,
+      "num_tokens": 16682.0,
+      "reward": 0.375,
+      "reward_std": 0.5386751294136047,
+      "rewards/reward_from_oracle/mean": 0.375,
+      "rewards/reward_from_oracle/std": 0.5175492167472839,
+      "step": 2
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 2.375,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 3.0,
+      "completions/max_terminated_length": 3.0,
+      "completions/mean_length": 2.375,
+      "completions/mean_terminated_length": 2.375,
+      "completions/min_length": 2.0,
+      "completions/min_terminated_length": 2.0,
+      "epoch": 0.1875,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 8.75147533416748,
+      "kl": 0.0,
+      "learning_rate": 4.849231551964771e-06,
+      "loss": 0.023432811722159386,
+      "num_tokens": 25025.0,
+      "reward": 0.25,
+      "reward_std": 0.5,
+      "rewards/reward_from_oracle/mean": 0.25,
+      "rewards/reward_from_oracle/std": 0.4629100561141968,
+      "step": 3
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 2.125,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 3.0,
+      "completions/max_terminated_length": 3.0,
+      "completions/mean_length": 2.125,
+      "completions/mean_terminated_length": 2.125,
+      "completions/min_length": 2.0,
+      "completions/min_terminated_length": 2.0,
+      "epoch": 0.25,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 10.478106498718262,
+      "kl": 0.0,
+      "learning_rate": 4.415111107797445e-06,
+      "loss": 0.013529304414987564,
+      "num_tokens": 33362.0,
+      "reward": 0.5,
+      "reward_std": 0.5773502588272095,
+      "rewards/reward_from_oracle/mean": 0.5,
+      "rewards/reward_from_oracle/std": 0.5345224738121033,
+      "step": 4
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 2.125,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 3.0,
+      "completions/max_terminated_length": 3.0,
+      "completions/mean_length": 2.125,
+      "completions/mean_terminated_length": 2.125,
+      "completions/min_length": 2.0,
+      "completions/min_terminated_length": 2.0,
+      "epoch": 0.3125,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 8.125267028808594,
+      "kl": 0.0,
+      "learning_rate": 3.7500000000000005e-06,
+      "loss": 0.013529304414987564,
+      "num_tokens": 41707.0,
+      "reward": 0.625,
+      "reward_std": 0.5386751294136047,
+      "rewards/reward_from_oracle/mean": 0.625,
+      "rewards/reward_from_oracle/std": 0.5175492167472839,
+      "step": 5
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 2.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 2.0,
+      "completions/max_terminated_length": 2.0,
+      "completions/mean_length": 2.0,
+      "completions/mean_terminated_length": 2.0,
+      "completions/min_length": 2.0,
+      "completions/min_terminated_length": 2.0,
+      "epoch": 0.375,
+      "frac_reward_zero_std": 0.5,
+      "grad_norm": 3.183867931365967,
+      "kl": 0.0,
+      "learning_rate": 2.9341204441673267e-06,
+      "loss": 0.0,
+      "num_tokens": 50047.0,
+      "reward": 0.875,
+      "reward_std": 0.25,
+      "rewards/reward_from_oracle/mean": 0.875,
+      "rewards/reward_from_oracle/std": 0.3535533845424652,
+      "step": 6
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 2.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 2.0,
+      "completions/max_terminated_length": 2.0,
+      "completions/mean_length": 2.0,
+      "completions/mean_terminated_length": 2.0,
+      "completions/min_length": 2.0,
+      "completions/min_terminated_length": 2.0,
+      "epoch": 0.4375,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 7.007436275482178,
+      "kl": 0.0,
+      "learning_rate": 2.0658795558326745e-06,
+      "loss": 0.0,
+      "num_tokens": 58391.0,
+      "reward": 0.75,
+      "reward_std": 0.5,
+      "rewards/reward_from_oracle/mean": 0.75,
+      "rewards/reward_from_oracle/std": 0.4629100561141968,
+      "step": 7
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 2.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 2.0,
+      "completions/max_terminated_length": 2.0,
+      "completions/mean_length": 2.0,
+      "completions/mean_terminated_length": 2.0,
+      "completions/min_length": 2.0,
+      "completions/min_terminated_length": 2.0,
+      "epoch": 0.5,
+      "frac_reward_zero_std": 0.5,
+      "grad_norm": 3.759775161743164,
+      "kl": 0.0,
+      "learning_rate": 1.2500000000000007e-06,
+      "loss": 1.862645149230957e-09,
+      "num_tokens": 66727.0,
+      "reward": 0.875,
+      "reward_std": 0.25,
+      "rewards/reward_from_oracle/mean": 0.875,
+      "rewards/reward_from_oracle/std": 0.3535533845424652,
+      "step": 8
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 2.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 2.0,
+      "completions/max_terminated_length": 2.0,
+      "completions/mean_length": 2.0,
+      "completions/mean_terminated_length": 2.0,
+      "completions/min_length": 2.0,
+      "completions/min_terminated_length": 2.0,
+      "epoch": 0.5625,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 7.748785495758057,
+      "kl": 0.0,
+      "learning_rate": 5.848888922025553e-07,
+      "loss": 1.862645149230957e-09,
+      "num_tokens": 75063.0,
+      "reward": 0.75,
+      "reward_std": 0.5,
+      "rewards/reward_from_oracle/mean": 0.75,
+      "rewards/reward_from_oracle/std": 0.4629100561141968,
+      "step": 9
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completion_length": 2.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 2.0,
+      "completions/max_terminated_length": 2.0,
+      "completions/mean_length": 2.0,
+      "completions/mean_terminated_length": 2.0,
+      "completions/min_length": 2.0,
+      "completions/min_terminated_length": 2.0,
+      "epoch": 0.625,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.0,
+      "kl": 0.0,
+      "learning_rate": 1.507684480352292e-07,
+      "loss": 0.0,
+      "num_tokens": 83403.0,
+      "reward": 1.0,
+      "reward_std": 0.0,
+      "rewards/reward_from_oracle/mean": 1.0,
+      "rewards/reward_from_oracle/std": 0.0,
+      "step": 10
+    }
+  ],
+  "logging_steps": 1,
+  "max_steps": 10,
+  "num_input_tokens_seen": 83403,
+  "num_train_epochs": 1,
+  "save_steps": 10,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 8,
+  "trial_name": null,
+  "trial_params": null
+}

scripts/start_space.sh

@@ -0,0 +1,54 @@
+#!/usr/bin/env bash
+set -eu
+
+export FREECIV_SERVER_URL="${FREECIV_SERVER_URL:-http://127.0.0.1}"
+export FREECIV_USERNAME="${FREECIV_USERNAME:-openenvbot}"
+export FREECIV_CLIENT_PORT="${FREECIV_CLIENT_PORT:-6000}"
+export FREECIV_TURN_TIMEOUT_S="${FREECIV_TURN_TIMEOUT_S:-120}"
+export FREECIV_STARTUP_TIMEOUT_S="${FREECIV_STARTUP_TIMEOUT_S:-180}"
+export ENABLE_WEB_INTERFACE="${ENABLE_WEB_INTERFACE:-true}"
+
+log_file=/tmp/start_space.log
+: > "$log_file"
+
+log() {
+    local line
+    line="[$(date -Iseconds)] $*"
+    echo "$line" | tee -a "$log_file" >&2
+}
+
+service_status() {
+    local name url
+    name="$1"
+    url="$2"
+    if curl -fsS --max-time 2 "$url" >/dev/null 2>&1; then
+        echo "$name=up"
+    else
+        echo "$name=down"
+    fi
+}
+
+wait_for_runtime() {
+    local deadline status nginx publite2 tomcat
+    deadline=$(( $(date +%s) + FREECIV_STARTUP_TIMEOUT_S ))
+    while true; do
+        nginx=$(service_status nginx http://127.0.0.1/)
+        publite2=$(service_status publite2 http://127.0.0.1/pubstatus)
+        tomcat=$(service_status tomcat http://127.0.0.1:8080/freeciv-web/)
+        status="$nginx $publite2 $tomcat"
+        log "$status"
+        if [[ "$nginx" == "nginx=up" && "$publite2" == "publite2=up" && "$tomcat" == "tomcat=up" ]]; then
+            return 0
+        fi
+        if (( $(date +%s) >= deadline )); then
+            log "freeciv runtime failed to become ready before timeout=${FREECIV_STARTUP_TIMEOUT_S}s"
+            return 1
+        fi
+        sleep 2
+    done
+}
+
+log "start_space.sh boot"
+wait_for_runtime
+log "freeciv runtime ready; starting uvicorn"
+exec python -m uvicorn server.app:app --host 0.0.0.0 --port 8000 --ws-ping-interval 300 --ws-ping-timeout 300