diff --git a/CLAUDE.md b/CLAUDE.md index 54e3f10eddd484ae9d96def4f1b25e0ad69a2cea..2f931e4a63a53461d227672ad58dba59252c6c49 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -28,7 +28,7 @@ Wire-protocol classes are template-locked: `CrisisworldcortexAction`, `Crisiswor - The four wire-protocol class names above. Do not rename. - Flat package layout: `pyproject.toml:package-dir` maps `CrisisWorldCortex` to repo root. Do not move `models.py`, `client.py`, or root `__init__.py` off root. -- Canonical wire-type imports in every `server/` module: `from CrisisWorldCortex.models import ...`. +- Dual-import fallback pattern (`try: from ..models / except: from models`) in every `server/` module that imports `models`. - `pyproject.toml`: deps, entry points, `packages`, `package-dir`. - `openenv.yaml` keys: `spec_version`, `runtime`, `app`, `port`. - `server/app.py:create_app(...)` call signature. @@ -47,7 +47,7 @@ Wire-protocol classes are template-locked: `CrisisworldcortexAction`, `Crisiswor ## Import-graph rule (enforced) - `cortex/` imports `models` and `cortex/*` only. -- `server/` imports `models`, `openenv.core.*`, and package-relative `server` internals only. No `cortex/*`, no `training/*`, no `baselines/*`, no `demo/*`. +- `server/` imports `models`, `openenv.core.*`, `server/*` only. No `cortex/*`, no `training/*`, no `baselines/*`, no `demo/*`. - `baselines/` imports `models`, `client`, `cortex/*`. Must not import `server/*` — baselines hit the env over HTTP like production. - `training/` imports `models`, `client`, `cortex/*`, `server.graders` (reward-name constants only). Must not import `server.simulator/*`. - `demo/` imports `cortex.schemas` (types only) and stdlib. Must not import `server/*`, `training/*`, `baselines/*`, `cortex.council`, `cortex.routing_policy`. @@ -70,9 +70,9 @@ Do not restate subsystem APIs in this file or in other subsystem files. - Wire package: `from CrisisWorldCortex import CrisisworldcortexAction, ...`. - Dev / research directories (sibling-of-repo-root): bare-name — `import cortex`, `import baselines`, `import training`, `import demo`, `import scripts`. -- Server-internal: package-relative imports such as `from .simulator import ...` or `from ..simulator import ...`; do not use `from CrisisWorldCortex.server...`. +- Server-internal: `from server.simulator import ...`, `from server.graders import ...`. - **Cross-boundary into wire types**: when a dev / research directory (`cortex/`, `baselines/`, `training/`, `demo/`) crosses into the wire-protocol package, use `from CrisisWorldCortex.models import …` — **never** bare `from models import …`. Bare-name only applies to dev-directory siblings and to same-subpackage imports inside `server/`. The dual-path import creates distinct `sys.modules` entries and breaks Pydantic discriminator checks across import boundaries (verified by session 4's class-identity bug; `cortex/schemas.py:21` documents the canonicalised import). -- **Cross-boundary into wire types from server modules**: every `server/` file imports wire types with `from CrisisWorldCortex.models import …`. Do not use `from ..models` or bare `from models`; both can create a second class identity under different launch modes and break Pydantic discriminated-union validation. +- **Cross-boundary into wire types from deep server modules**: files inside `server/` more than one level deep (e.g., `server/simulator/seir_model.py`, `server/graders/outer_reward.py`) cannot use the `try: from ..models / except: from models` fallback — `..models` from a two-level-deep module resolves to a non-existent `CrisisWorldCortex.server.models`, the fallback fires, and bare `models` loads as a separate `sys.modules` entry. Use the absolute path: `from CrisisWorldCortex.models import …`. Session 4 (`cortex/schemas.py:21`) and Session 5a (`server/simulator/seir_model.py`, `server/simulator/tasks.py`) document this with inline comments. The dual-import fallback in `server/CrisisWorldCortex_environment.py` and `server/app.py` works only because they are one level deep (`..models` → `CrisisWorldCortex.models` directly). ## Commands (Git Bash; quote Windows paths) diff --git a/README.md b/README.md index 8301f878ed0515fff2cedf566f379524dc2bc6c5..10f5b94f0b0a1b436e53a28e39fbdb6d423e2868 100644 --- a/README.md +++ b/README.md @@ -1,255 +1,255 @@ ---- -title: Crisisworldcortex Environment Server -emoji: 🌟 -colorFrom: yellow -colorTo: pink -sdk: docker -pinned: false -app_port: 8000 -base_path: /web -tags: - - openenv ---- - -# Crisisworldcortex Environment - -A simple test environment that echoes back messages. Perfect for testing the env APIs as well as demonstrating environment usage patterns. - -## Quick Start - -The simplest way to use the Crisisworldcortex environment is through the `CrisisworldcortexEnv` class: - -```python -from CrisisWorldCortex import CrisisworldcortexAction, CrisisworldcortexEnv - -try: - # Create environment from Docker image - CrisisWorldCortexenv = CrisisworldcortexEnv.from_docker_image("CrisisWorldCortex-env:latest") - - # Reset - result = CrisisWorldCortexenv.reset() - print(f"Reset: {result.observation.echoed_message}") - - # Send multiple messages - messages = ["Hello, World!", "Testing echo", "Final message"] - - for msg in messages: - result = CrisisWorldCortexenv.step(CrisisworldcortexAction(message=msg)) - print(f"Sent: '{msg}'") - print(f" → Echoed: '{result.observation.echoed_message}'") - print(f" → Length: {result.observation.message_length}") - print(f" → Reward: {result.reward}") - -finally: - # Always clean up - CrisisWorldCortexenv.close() -``` - -That's it! The `CrisisworldcortexEnv.from_docker_image()` method handles: -- Starting the Docker container -- Waiting for the server to be ready -- Connecting to the environment -- Container cleanup when you call `close()` - -## Building the Docker Image - -Before using the environment, you need to build the Docker image: - -```bash -# From project root -docker build -t CrisisWorldCortex-env:latest -f server/Dockerfile . -``` - -## Deploying to Hugging Face Spaces - -You can easily deploy your OpenEnv environment to Hugging Face Spaces using the `openenv push` command: - -```bash -# From the environment directory (where openenv.yaml is located) -openenv push - -# Or specify options -openenv push --namespace my-org --private -``` - -The `openenv push` command will: -1. Validate that the directory is an OpenEnv environment (checks for `openenv.yaml`) -2. Prepare a custom build for Hugging Face Docker space (enables web interface) -3. Upload to Hugging Face (ensuring you're logged in) - -### Prerequisites - -- Authenticate with Hugging Face: The command will prompt for login if not already authenticated - -### Options - -- `--directory`, `-d`: Directory containing the OpenEnv environment (defaults to current directory) -- `--repo-id`, `-r`: Repository ID in format 'username/repo-name' (defaults to 'username/env-name' from openenv.yaml) -- `--base-image`, `-b`: Base Docker image to use (overrides Dockerfile FROM) -- `--private`: Deploy the space as private (default: public) - -### Examples - -```bash -# Push to your personal namespace (defaults to username/env-name from openenv.yaml) -openenv push - -# Push to a specific repository -openenv push --repo-id my-org/my-env - -# Push with a custom base image -openenv push --base-image ghcr.io/meta-pytorch/openenv-base:latest - -# Push as a private space -openenv push --private - -# Combine options -openenv push --repo-id my-org/my-env --base-image custom-base:latest --private -``` - -After deployment, your space will be available at: -`https://huggingface.co/spaces/` - -The deployed space includes: -- **Web Interface** at `/web` - Interactive UI for exploring the environment -- **API Documentation** at `/docs` - Full OpenAPI/Swagger interface -- **Health Check** at `/health` - Container health monitoring -- **WebSocket** at `/ws` - Persistent session endpoint for low-latency interactions - -## Environment Details - -### Action -**CrisisworldcortexAction**: Contains a single field -- `message` (str) - The message to echo back - -### Observation -**CrisisworldcortexObservation**: Contains the echo response and metadata -- `echoed_message` (str) - The message echoed back -- `message_length` (int) - Length of the message -- `reward` (float) - Reward based on message length (length × 0.1) -- `done` (bool) - Always False for echo environment -- `metadata` (dict) - Additional info like step count - -### Reward -The reward is calculated as: `message_length × 0.1` -- "Hi" → reward: 0.2 -- "Hello, World!" → reward: 1.3 -- Empty message → reward: 0.0 - -## Advanced Usage - -### Connecting to an Existing Server - -If you already have a Crisisworldcortex environment server running, you can connect directly: - -```python -from CrisisWorldCortex import CrisisworldcortexEnv - -# Connect to existing server -CrisisWorldCortexenv = CrisisworldcortexEnv(base_url="") - -# Use as normal -result = CrisisWorldCortexenv.reset() -result = CrisisWorldCortexenv.step(CrisisworldcortexAction(message="Hello!")) -``` - -Note: When connecting to an existing server, `CrisisWorldCortexenv.close()` will NOT stop the server. - -### Using the Context Manager - -The client supports context manager usage for automatic connection management: - -```python -from CrisisWorldCortex import CrisisworldcortexAction, CrisisworldcortexEnv - -# Connect with context manager (auto-connects and closes) -with CrisisworldcortexEnv(base_url="http://localhost:8000") as env: - result = env.reset() - print(f"Reset: {result.observation.echoed_message}") - # Multiple steps with low latency - for msg in ["Hello", "World", "!"]: - result = env.step(CrisisworldcortexAction(message=msg)) - print(f"Echoed: {result.observation.echoed_message}") -``` - -The client uses WebSocket connections for: -- **Lower latency**: No HTTP connection overhead per request -- **Persistent session**: Server maintains your environment state -- **Efficient for episodes**: Better for many sequential steps - -### Concurrent WebSocket Sessions - -The server supports multiple concurrent WebSocket connections. To enable this, -modify `server/app.py` to use factory mode: - -```python -# In server/app.py - use factory mode for concurrent sessions -app = create_app( - CrisisworldcortexEnvironment, # Pass class, not instance - CrisisworldcortexAction, - CrisisworldcortexObservation, - max_concurrent_envs=4, # Allow 4 concurrent sessions -) -``` - -Then multiple clients can connect simultaneously: - -```python -from CrisisWorldCortex import CrisisworldcortexAction, CrisisworldcortexEnv -from concurrent.futures import ThreadPoolExecutor - -def run_episode(client_id: int): - with CrisisworldcortexEnv(base_url="http://localhost:8000") as env: - result = env.reset() - for i in range(10): - result = env.step(CrisisworldcortexAction(message=f"Client {client_id}, step {i}")) - return client_id, result.observation.message_length - -# Run 4 episodes concurrently -with ThreadPoolExecutor(max_workers=4) as executor: - results = list(executor.map(run_episode, range(4))) -``` - -## Development & Testing - -### Direct Environment Testing - -Test the environment logic directly without starting the HTTP server: - -```bash -# From the server directory -python3 server/CrisisWorldCortex_environment.py -``` - -This verifies that: -- Environment resets correctly -- Step executes actions properly -- State tracking works -- Rewards are calculated correctly - -### Running Locally - -Run the server locally for development: - -```bash -uvicorn server.app:app --reload -``` - -## Project Structure - -``` -CrisisWorldCortex/ -├── .dockerignore # Docker build exclusions -├── __init__.py # Module exports -├── README.md # This file -├── openenv.yaml # OpenEnv manifest -├── pyproject.toml # Project metadata and dependencies -├── uv.lock # Locked dependencies (generated) -├── client.py # CrisisworldcortexEnv client -├── models.py # Action and Observation models -└── server/ - ├── __init__.py # Server module exports - ├── CrisisWorldCortex_environment.py # Core environment logic - ├── app.py # FastAPI application (HTTP + WebSocket endpoints) - └── Dockerfile # Container image definition -``` +--- +title: Crisisworldcortex Environment Server +emoji: 🌟 +colorFrom: yellow +colorTo: pink +sdk: docker +pinned: false +app_port: 8000 +base_path: /web +tags: + - openenv +--- + +# Crisisworldcortex Environment + +A simple test environment that echoes back messages. Perfect for testing the env APIs as well as demonstrating environment usage patterns. + +## Quick Start + +The simplest way to use the Crisisworldcortex environment is through the `CrisisworldcortexEnv` class: + +```python +from CrisisWorldCortex import CrisisworldcortexAction, CrisisworldcortexEnv + +try: + # Create environment from Docker image + CrisisWorldCortexenv = CrisisworldcortexEnv.from_docker_image("CrisisWorldCortex-env:latest") + + # Reset + result = CrisisWorldCortexenv.reset() + print(f"Reset: {result.observation.echoed_message}") + + # Send multiple messages + messages = ["Hello, World!", "Testing echo", "Final message"] + + for msg in messages: + result = CrisisWorldCortexenv.step(CrisisworldcortexAction(message=msg)) + print(f"Sent: '{msg}'") + print(f" → Echoed: '{result.observation.echoed_message}'") + print(f" → Length: {result.observation.message_length}") + print(f" → Reward: {result.reward}") + +finally: + # Always clean up + CrisisWorldCortexenv.close() +``` + +That's it! The `CrisisworldcortexEnv.from_docker_image()` method handles: +- Starting the Docker container +- Waiting for the server to be ready +- Connecting to the environment +- Container cleanup when you call `close()` + +## Building the Docker Image + +Before using the environment, you need to build the Docker image: + +```bash +# From project root +docker build -t CrisisWorldCortex-env:latest -f server/Dockerfile . +``` + +## Deploying to Hugging Face Spaces + +You can easily deploy your OpenEnv environment to Hugging Face Spaces using the `openenv push` command: + +```bash +# From the environment directory (where openenv.yaml is located) +openenv push + +# Or specify options +openenv push --namespace my-org --private +``` + +The `openenv push` command will: +1. Validate that the directory is an OpenEnv environment (checks for `openenv.yaml`) +2. Prepare a custom build for Hugging Face Docker space (enables web interface) +3. Upload to Hugging Face (ensuring you're logged in) + +### Prerequisites + +- Authenticate with Hugging Face: The command will prompt for login if not already authenticated + +### Options + +- `--directory`, `-d`: Directory containing the OpenEnv environment (defaults to current directory) +- `--repo-id`, `-r`: Repository ID in format 'username/repo-name' (defaults to 'username/env-name' from openenv.yaml) +- `--base-image`, `-b`: Base Docker image to use (overrides Dockerfile FROM) +- `--private`: Deploy the space as private (default: public) + +### Examples + +```bash +# Push to your personal namespace (defaults to username/env-name from openenv.yaml) +openenv push + +# Push to a specific repository +openenv push --repo-id my-org/my-env + +# Push with a custom base image +openenv push --base-image ghcr.io/meta-pytorch/openenv-base:latest + +# Push as a private space +openenv push --private + +# Combine options +openenv push --repo-id my-org/my-env --base-image custom-base:latest --private +``` + +After deployment, your space will be available at: +`https://huggingface.co/spaces/` + +The deployed space includes: +- **Web Interface** at `/web` - Interactive UI for exploring the environment +- **API Documentation** at `/docs` - Full OpenAPI/Swagger interface +- **Health Check** at `/health` - Container health monitoring +- **WebSocket** at `/ws` - Persistent session endpoint for low-latency interactions + +## Environment Details + +### Action +**CrisisworldcortexAction**: Contains a single field +- `message` (str) - The message to echo back + +### Observation +**CrisisworldcortexObservation**: Contains the echo response and metadata +- `echoed_message` (str) - The message echoed back +- `message_length` (int) - Length of the message +- `reward` (float) - Reward based on message length (length × 0.1) +- `done` (bool) - Always False for echo environment +- `metadata` (dict) - Additional info like step count + +### Reward +The reward is calculated as: `message_length × 0.1` +- "Hi" → reward: 0.2 +- "Hello, World!" → reward: 1.3 +- Empty message → reward: 0.0 + +## Advanced Usage + +### Connecting to an Existing Server + +If you already have a Crisisworldcortex environment server running, you can connect directly: + +```python +from CrisisWorldCortex import CrisisworldcortexEnv + +# Connect to existing server +CrisisWorldCortexenv = CrisisworldcortexEnv(base_url="") + +# Use as normal +result = CrisisWorldCortexenv.reset() +result = CrisisWorldCortexenv.step(CrisisworldcortexAction(message="Hello!")) +``` + +Note: When connecting to an existing server, `CrisisWorldCortexenv.close()` will NOT stop the server. + +### Using the Context Manager + +The client supports context manager usage for automatic connection management: + +```python +from CrisisWorldCortex import CrisisworldcortexAction, CrisisworldcortexEnv + +# Connect with context manager (auto-connects and closes) +with CrisisworldcortexEnv(base_url="http://localhost:8000") as env: + result = env.reset() + print(f"Reset: {result.observation.echoed_message}") + # Multiple steps with low latency + for msg in ["Hello", "World", "!"]: + result = env.step(CrisisworldcortexAction(message=msg)) + print(f"Echoed: {result.observation.echoed_message}") +``` + +The client uses WebSocket connections for: +- **Lower latency**: No HTTP connection overhead per request +- **Persistent session**: Server maintains your environment state +- **Efficient for episodes**: Better for many sequential steps + +### Concurrent WebSocket Sessions + +The server supports multiple concurrent WebSocket connections. To enable this, +modify `server/app.py` to use factory mode: + +```python +# In server/app.py - use factory mode for concurrent sessions +app = create_app( + CrisisworldcortexEnvironment, # Pass class, not instance + CrisisworldcortexAction, + CrisisworldcortexObservation, + max_concurrent_envs=4, # Allow 4 concurrent sessions +) +``` + +Then multiple clients can connect simultaneously: + +```python +from CrisisWorldCortex import CrisisworldcortexAction, CrisisworldcortexEnv +from concurrent.futures import ThreadPoolExecutor + +def run_episode(client_id: int): + with CrisisworldcortexEnv(base_url="http://localhost:8000") as env: + result = env.reset() + for i in range(10): + result = env.step(CrisisworldcortexAction(message=f"Client {client_id}, step {i}")) + return client_id, result.observation.message_length + +# Run 4 episodes concurrently +with ThreadPoolExecutor(max_workers=4) as executor: + results = list(executor.map(run_episode, range(4))) +``` + +## Development & Testing + +### Direct Environment Testing + +Test the environment logic directly without starting the HTTP server: + +```bash +# From the server directory +python3 server/CrisisWorldCortex_environment.py +``` + +This verifies that: +- Environment resets correctly +- Step executes actions properly +- State tracking works +- Rewards are calculated correctly + +### Running Locally + +Run the server locally for development: + +```bash +uvicorn server.app:app --reload +``` + +## Project Structure + +``` +CrisisWorldCortex/ +├── .dockerignore # Docker build exclusions +├── __init__.py # Module exports +├── README.md # This file +├── openenv.yaml # OpenEnv manifest +├── pyproject.toml # Project metadata and dependencies +├── uv.lock # Locked dependencies (generated) +├── client.py # CrisisworldcortexEnv client +├── models.py # Action and Observation models +└── server/ + ├── __init__.py # Server module exports + ├── CrisisWorldCortex_environment.py # Core environment logic + ├── app.py # FastAPI application (HTTP + WebSocket endpoints) + └── Dockerfile # Container image definition +``` diff --git a/__init__.py b/__init__.py index 91c4e9a1e8762f878a688d9514f3af828f99c6a0..8a4a5706c4f95aa98625924e4b0b386e342ec274 100644 --- a/__init__.py +++ b/__init__.py @@ -1,16 +1,16 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# All rights reserved. -# -# This source code is licensed under the BSD-style license found in the -# LICENSE file in the root directory of this source tree. - -"""Crisisworldcortex Environment.""" - -from .client import CrisisworldcortexEnv -from .models import CrisisworldcortexAction, CrisisworldcortexObservation - -__all__ = [ - "CrisisworldcortexAction", - "CrisisworldcortexObservation", - "CrisisworldcortexEnv", -] +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +"""Crisisworldcortex Environment.""" + +from .client import CrisisworldcortexEnv +from .models import CrisisworldcortexAction, CrisisworldcortexObservation + +__all__ = [ + "CrisisworldcortexAction", + "CrisisworldcortexObservation", + "CrisisworldcortexEnv", +] diff --git a/cortex/brains/__init__.py b/cortex/brains/__init__.py index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..d4d3f7729ee42c7fca853c927ed37431503ebad3 100644 --- a/cortex/brains/__init__.py +++ b/cortex/brains/__init__.py @@ -0,0 +1,26 @@ +"""Cortex brains (Session 11+). + +Public surface: + - Brain: per-brain class wiring Perception + Lens + 3 LLM subagents + Brain Executive. + - EpiBrain, LogisticsBrain, GovernanceBrain: factory functions. + - aggregate_brain_outputs: Brain Executive aggregation function. + +Each Brain holds its own LLMClient instance; the orchestration layer +(Council Executive in Session 12, Workstream B trainers) constructs one +Brain per brain id, optionally with different LLMClients pointing to +different models. NO module-level state. +""" + +from ._base import Brain +from ._executive import aggregate_brain_outputs +from .epidemiology import EpiBrain +from .governance import GovernanceBrain +from .logistics import LogisticsBrain + +__all__ = [ + "Brain", + "EpiBrain", + "GovernanceBrain", + "LogisticsBrain", + "aggregate_brain_outputs", +] diff --git a/cortex/brains/_base.py b/cortex/brains/_base.py new file mode 100644 index 0000000000000000000000000000000000000000..7d93c6abedd7488a84fdb2a4cf086bdad05f0209 --- /dev/null +++ b/cortex/brains/_base.py @@ -0,0 +1,201 @@ +"""Brain class - composes Perception + Lens + 3 Subagents + Brain Executive. + +Per cortex/CLAUDE.md: each brain has a deterministic Python Perception ++ Lens, three LLM subagents (router-callable), and a deterministic +Python Brain Executive. The Brain class wires these together. + +Multi-model deployment: each Brain holds a SINGLE LLMClient instance +passed at construction. Different brains can use different models by +constructing each Brain with a different LLMClient (e.g., Qwen for epi, +Llama for logistics). NO module-level state, NO shared singletons. +""" + +from __future__ import annotations + +from typing import List, Literal + +from cortex.lenses import lens_for +from cortex.schemas import ( + BeliefState, + BrainLensedObservation, + BrainRecommendation, + CandidatePlan, + CriticReport, + PerceptionReport, + SubagentInput, +) +from cortex.subagents import ( + CriticSubagent, + PlannerSubagent, + WorldModelerSubagent, + perception_for, +) +from cortex.subagents._base import _LLMClientLike +from CrisisWorldCortex.models import CrisisworldcortexObservation + +from ._executive import aggregate_brain_outputs + +_BrainId = Literal["epidemiology", "logistics", "governance"] + + +class Brain: + """Per-brain pipeline holder. + + Each Brain instance owns its own LLMClient. The orchestration layer + (Session 12 Council, Workstream B trainers) instantiates one Brain + per brain id, optionally with different LLMClients pointing to + different models. The Brain class itself has NO module-level state + and NO forced singleton. + + Args: + brain_id: One of "epidemiology", "logistics", "governance". + llm_client: This brain's LLM client. Subagents are constructed + with the SAME client so token billing aggregates correctly. + wm: WorldModeler subagent. + planner: Planner subagent. + critic: Critic subagent. + """ + + def __init__( + self, + brain_id: _BrainId, + llm_client: _LLMClientLike, + wm: WorldModelerSubagent, + planner: PlannerSubagent, + critic: CriticSubagent, + ) -> None: + self.brain_id = brain_id + self.llm_client = llm_client + self.wm = wm + self.planner = planner + self.critic = critic + + # ------------------------------------------------------------------ + # Deterministic Python pieces (no LLM) + # ------------------------------------------------------------------ + + def compute_perception(self, obs: CrisisworldcortexObservation) -> PerceptionReport: + """Run this brain's Perception. Pure Python; no LLM.""" + return perception_for(self.brain_id, obs) + + def compute_lens( + self, obs: CrisisworldcortexObservation, last_reward: float + ) -> BrainLensedObservation: + """Run this brain's Lens. Pure Python; no LLM.""" + return lens_for(self.brain_id, obs, last_reward) + + def aggregate( + self, + perception: PerceptionReport, + beliefs: List[BeliefState], + plans: List[CandidatePlan], + critics: List[CriticReport], + tokens_used: int = 0, + ) -> BrainRecommendation: + """Run this brain's Brain Executive. Pure Python; no LLM.""" + return aggregate_brain_outputs( + brain_id=self.brain_id, + perception=perception, + beliefs=beliefs, + plans=plans, + critics=critics, + tokens_used=tokens_used, + ) + + # ------------------------------------------------------------------ + # High-level convenience: round-1 single tick + # ------------------------------------------------------------------ + + def run_tick( + self, + obs: CrisisworldcortexObservation, + last_reward: float, + tick: int, + round_: int = 1, + ) -> BrainRecommendation: + """Round-1 single-tick pipeline (Session 11 smoke). + + Round 2 is orchestrated by the Council Executive (Session 12) + via the fine-grained methods (compute_perception, compute_lens, + wm.run / planner.run / critic.run, aggregate). Calling this + convenience method with ``round_!=1`` raises NotImplementedError + to prevent accidental misuse before the Council exists. + """ + if round_ != 1: + raise NotImplementedError( + f"Round {round_} orchestration is the Council Executive's " + f"responsibility (Session 12). Use Brain.compute_perception/" + f"compute_lens + WorldModelerSubagent.run/PlannerSubagent.run/" + f"CriticSubagent.run + Brain.aggregate directly." + ) + + perception = self.compute_perception(obs) + # Lens is computed for completeness; Session 11 doesn't yet plumb + # it into SubagentInput (M-FR-4 step indices fixed). Session 12 + # Council will extend the SubagentInput contract to carry lens + # output if subagents need it. + _ = self.compute_lens(obs, last_reward) + + # WorldModeler (step_idx=0) + wm_input = SubagentInput( + brain=self.brain_id, + role="world_modeler", + tick=tick, + round=round_, + perception=perception, + prior_belief=None, + prior_plans=[], + target_plan_id=None, + last_reward=last_reward, + recent_action_log_excerpt=list(obs.recent_action_log), + ) + belief = self.wm.run(wm_input, step_idx=0) + + # Planner (step_idx=1) + planner_input = SubagentInput( + brain=self.brain_id, + role="planner", + tick=tick, + round=round_, + perception=perception, + prior_belief=belief, + prior_plans=[], + target_plan_id=None, + last_reward=last_reward, + recent_action_log_excerpt=list(obs.recent_action_log), + ) + plan = self.planner.run(planner_input, step_idx=1) + + # Critic (step_idx=2) + critic_input = SubagentInput( + brain=self.brain_id, + role="critic", + tick=tick, + round=round_, + perception=perception, + prior_belief=belief, + prior_plans=[plan], + target_plan_id="plan-0", + last_reward=last_reward, + recent_action_log_excerpt=list(obs.recent_action_log), + ) + critic = self.critic.run(critic_input, step_idx=2) + + # Tally tokens billed to this brain's caller_ids. + caller_id_base = f"cortex:{self.brain_id}" + tokens_used = sum( + self.llm_client.tokens_used_for(f"{caller_id_base}:{role}:t{tick}:r{round_}:s{idx}") + for role, idx in ( + ("world_modeler", 0), + ("planner", 1), + ("critic", 2), + ) + ) + + return self.aggregate( + perception=perception, + beliefs=[belief], + plans=[plan], + critics=[critic], + tokens_used=tokens_used, + ) diff --git a/cortex/brains/_executive.py b/cortex/brains/_executive.py new file mode 100644 index 0000000000000000000000000000000000000000..8f55c1e73c6d2d89ae01459e212cedfa0b375104 --- /dev/null +++ b/cortex/brains/_executive.py @@ -0,0 +1,144 @@ +"""Brain Executive - deterministic Python aggregation. + +Per Phase A docs/CORTEX_ARCHITECTURE.md Decisions 15-21 + M-FR-3 partial +evidence union (perception + beliefs only; CandidatePlan and CriticReport +schemas have no evidence fields). + +Brain Executive runs ONCE per brain at round end. NOT router-callable +per cortex/CLAUDE.md. +""" + +from __future__ import annotations + +from typing import List + +from cortex.schemas import ( + BeliefState, + BrainRecommendation, + CandidatePlan, + CriticReport, + EvidenceCitation, + PerceptionReport, +) +from CrisisWorldCortex.models import NoOp + +_REASONING_SUMMARY_MAX_CHARS = 400 # matches BrainRecommendation.reasoning_summary cap +_FALSIFIERS_TO_JOIN = 3 +_FALSIFIER_FALLBACK = "(no falsifier provided)" +_EMPTY_REASONING = "(empty: no subagent produced a parseable plan)" + + +def aggregate_brain_outputs( + brain_id: str, + perception: PerceptionReport, + beliefs: List[BeliefState], + plans: List[CandidatePlan], + critics: List[CriticReport], + tokens_used: int = 0, +) -> BrainRecommendation: + """Aggregate one brain's per-round subagent outputs into a recommendation. + + Decisions: + D15: argmax over expected_value * confidence. + D16: top_confidence = chosen.confidence * (1 - chosen_belief.uncertainty). + D17: minority_actions = all expected_outer_actions except chosen. + D19: reasoning_summary = chosen.action_sketch[:400]. + D20 (M-FR-3): evidence = perception.evidence + flat-union of beliefs[*].evidence. + CandidatePlan/CriticReport carry no evidence fields. + D21: brain_id is lowercase per Pydantic Literal in BrainRecommendation. + + Empty fallback (M-FR-7): no plans, or chosen plan has confidence==0 + -> top_action=NoOp, top_confidence=0, uncertainty=1.0, + reasoning_summary=_EMPTY_REASONING. + + Args: + brain_id: lowercase brain id ("epidemiology" / "logistics" / "governance"). + perception: This brain's PerceptionReport for the tick. + beliefs: Per-round BeliefStates. Index aligned with ``plans``. + plans: Per-round CandidatePlans. + critics: Per-round CriticReports (currently unused in aggregation but + kept on the signature so the trajectory log captures the full + chain). + tokens_used: Total tokens billed across this brain's subagents. + """ + if not plans: + return _empty_recommendation(brain_id, perception, beliefs, tokens_used) + + # D15: argmax over expected_value * confidence + chosen_idx = max( + range(len(plans)), + key=lambda i: plans[i].expected_value * plans[i].confidence, + ) + chosen_plan = plans[chosen_idx] + + if chosen_plan.confidence == 0.0: + # All plans are empty fallbacks (or the only plan is empty). + # Brain Executive treats this as no-signal. + return _empty_recommendation(brain_id, perception, beliefs, tokens_used) + + # D16: top_confidence = chosen.confidence * (1 - belief.uncertainty) + if chosen_idx < len(beliefs): + chosen_belief = beliefs[chosen_idx] + uncertainty = chosen_belief.uncertainty + else: + # Defensive: parallel arrays should match. If not, treat as max uncertainty. + uncertainty = 1.0 + top_confidence = chosen_plan.confidence * (1.0 - uncertainty) + + # D17: minority_actions = all plans except chosen + minority_actions = [ + plans[i].expected_outer_action for i in range(len(plans)) if i != chosen_idx + ] + + # D19: reasoning_summary + reasoning_summary = chosen_plan.action_sketch[:_REASONING_SUMMARY_MAX_CHARS] + + # D20 (M-FR-3): evidence union from perception + beliefs only. + # CandidatePlan and CriticReport schemas (Session 9) have no evidence fields; + # the perception+beliefs union captures the actionable evidence chain since + # plans/critics derive from beliefs. + evidence: List[EvidenceCitation] = list(perception.evidence) + for b in beliefs: + evidence.extend(b.evidence) + + # falsifier (M-FR-6): join up to 3 falsifiers; fallback if empty. + if chosen_plan.falsifiers: + falsifier = "; ".join(chosen_plan.falsifiers[:_FALSIFIERS_TO_JOIN]) + else: + falsifier = _FALSIFIER_FALLBACK + + return BrainRecommendation( + brain=brain_id, + top_action=chosen_plan.expected_outer_action, + top_confidence=top_confidence, + minority_actions=minority_actions, + reasoning_summary=reasoning_summary, + evidence=evidence, + falsifier=falsifier, + uncertainty=uncertainty, + tokens_used=tokens_used, + ) + + +def _empty_recommendation( + brain_id: str, + perception: PerceptionReport, + beliefs: List[BeliefState], + tokens_used: int, +) -> BrainRecommendation: + """M-FR-7 empty fallback: NoOp + confidence=0 + uncertainty=1.""" + evidence: List[EvidenceCitation] = list(perception.evidence) + for b in beliefs: + evidence.extend(b.evidence) + + return BrainRecommendation( + brain=brain_id, + top_action=NoOp(), + top_confidence=0.0, + minority_actions=[], + reasoning_summary=_EMPTY_REASONING, + evidence=evidence, + falsifier=_FALSIFIER_FALLBACK, + uncertainty=1.0, + tokens_used=tokens_used, + ) diff --git a/cortex/brains/epidemiology.py b/cortex/brains/epidemiology.py new file mode 100644 index 0000000000000000000000000000000000000000..a899111a737143b308aacb3bde30715a93806f11 --- /dev/null +++ b/cortex/brains/epidemiology.py @@ -0,0 +1,25 @@ +"""Epidemiology brain factory.""" + +from __future__ import annotations + +from cortex.subagents import CriticSubagent, PlannerSubagent, WorldModelerSubagent +from cortex.subagents._base import _LLMClientLike + +from ._base import Brain + + +def EpiBrain(llm_client: _LLMClientLike) -> Brain: + """Construct an Epidemiology Brain bound to ``llm_client``. + + Multi-model deployment: pass a different ``llm_client`` per brain + instance to use different models per brain (e.g., Qwen for epi, + Llama for logistics). The 3 LLM subagents are constructed with the + SAME client so token billing aggregates correctly. + """ + return Brain( + brain_id="epidemiology", + llm_client=llm_client, + wm=WorldModelerSubagent(llm_client), + planner=PlannerSubagent(llm_client), + critic=CriticSubagent(llm_client), + ) diff --git a/cortex/brains/governance.py b/cortex/brains/governance.py new file mode 100644 index 0000000000000000000000000000000000000000..4ec8382e6764d94601e906ac6ae4e5cc9c6683e7 --- /dev/null +++ b/cortex/brains/governance.py @@ -0,0 +1,22 @@ +"""Governance brain factory.""" + +from __future__ import annotations + +from cortex.subagents import CriticSubagent, PlannerSubagent, WorldModelerSubagent +from cortex.subagents._base import _LLMClientLike + +from ._base import Brain + + +def GovernanceBrain(llm_client: _LLMClientLike) -> Brain: + """Construct a Governance Brain bound to ``llm_client``. + + Multi-model deployment: see EpiBrain. + """ + return Brain( + brain_id="governance", + llm_client=llm_client, + wm=WorldModelerSubagent(llm_client), + planner=PlannerSubagent(llm_client), + critic=CriticSubagent(llm_client), + ) diff --git a/cortex/brains/logistics.py b/cortex/brains/logistics.py new file mode 100644 index 0000000000000000000000000000000000000000..857ea5fcfafb0329c924279bcb866ccf9a4497cd --- /dev/null +++ b/cortex/brains/logistics.py @@ -0,0 +1,23 @@ +"""Logistics brain factory.""" + +from __future__ import annotations + +from cortex.subagents import CriticSubagent, PlannerSubagent, WorldModelerSubagent +from cortex.subagents._base import _LLMClientLike + +from ._base import Brain + + +def LogisticsBrain(llm_client: _LLMClientLike) -> Brain: + """Construct a Logistics Brain bound to ``llm_client``. + + Multi-model deployment: see EpiBrain. Pass a Llama-bound client + here while the other brains use Qwen, etc. + """ + return Brain( + brain_id="logistics", + llm_client=llm_client, + wm=WorldModelerSubagent(llm_client), + planner=PlannerSubagent(llm_client), + critic=CriticSubagent(llm_client), + ) diff --git a/cortex/lenses.py b/cortex/lenses.py new file mode 100644 index 0000000000000000000000000000000000000000..ca7820fdf21c20053c0d1f53637fd4b62c255225 --- /dev/null +++ b/cortex/lenses.py @@ -0,0 +1,186 @@ +"""Brain-specific observation lenses (Session 10). + +Phase A docs/CORTEX_ARCHITECTURE.md Decisions 9-14 + §2 A1. + +Lenses are pure-Python: no LLM, no I/O, no state. ``lens_for(brain, obs, +last_reward)`` dispatches to one of 3 brain-specific helpers and returns +a ``BrainLensedObservation``. V2 brain ids raise ``KeyError`` per +Decision 9 (post-review) -- no MVP stub functions. + +The lens does NOT strip the raw observation (Decision 13); subagents +may need fields the lens didn't emphasise. ``salient_field_ids`` is a +salience map alongside ``raw_obs``, not a replacement for it. + +``transmission_rate_trend`` is fixed at 0.0 in MVP (M-FR-2): the lens +sees one observation per call. Session 11 plumbs prior-tick obs into +the lens to enable real trend computation. +""" + +from __future__ import annotations + +from typing import Callable, Dict + +from cortex.schemas import BrainLensedObservation +from CrisisWorldCortex.models import CrisisworldcortexObservation + +_V2_BRAINS = frozenset({"communications", "equity"}) + + +def lens_for( + brain: str, + obs: CrisisworldcortexObservation, + last_reward: float, +) -> BrainLensedObservation: + """Return the per-brain lensed observation. + + Args: + brain: One of {"epidemiology", "logistics", "governance"}. + obs: The current tick's observation. + last_reward: Previous tick's reward (plumbed from B1's pattern, + included on the lensed object so all 3 subagents in this + brain see the same recency signal). + + Raises: + KeyError: If ``brain`` is a V2-deferred brain (communications, + equity) or unknown. + """ + helper = _LENS_REGISTRY.get(brain) + if helper is not None: + return helper(obs, last_reward) + if brain in _V2_BRAINS: + raise KeyError( + f"V2 brain {brain!r} deferred per Phase A Decision 9; " + f"no MVP stub lens. See docs/CORTEX_ARCHITECTURE.md." + ) + raise KeyError(f"unknown brain: {brain!r}") + + +# ============================================================================ +# Per-brain lens helpers +# ============================================================================ + + +def _epi_lens(obs: CrisisworldcortexObservation, last_reward: float) -> BrainLensedObservation: + """Epidemiology lens (Decision 10, M-FR-4 rename: epi_pressure).""" + n_regions = max(1, len(obs.regions)) + mean_hospital_load = sum(r.hospital_load for r in obs.regions) / n_regions + # M-FR-4: pressure scalar correlated with R_eff but not a true R_eff + # estimate. WorldModeler subagent computes proper R_eff during reasoning. + epi_pressure = max(0.0, min(3.0, mean_hospital_load * 2.0)) + + max_cases = max((r.reported_cases_d_ago for r in obs.regions), default=0) + # /1000 normaliser matches the design-doc "~30 cases / 1000 pop" spec + worst_region_infection = max(0.0, min(1.0, max_cases / 1000.0)) + + return BrainLensedObservation( + brain="epidemiology", + raw_obs=obs, + salient_field_ids=[ + "regions[*].reported_cases_d_ago", + "regions[*].hospital_load", + "regions[*].compliance_proxy", + ], + derived_features={ + "epi_pressure": float(epi_pressure), + "worst_region_infection": float(worst_region_infection), + # M-FR-2: needs history; Session 11 plumbs prior-tick obs. + "transmission_rate_trend": 0.0, + }, + last_reward=last_reward, + ) + + +def _logistics_lens( + obs: CrisisworldcortexObservation, last_reward: float +) -> BrainLensedObservation: + """Logistics lens (Decision 11, M-FR-3 floor 0.5, M-FR-6 flat keys).""" + res = obs.resources + total_inventory = float( + res.test_kits + res.hospital_beds_free + res.mobile_units + res.vaccine_doses + ) + + hospital_load_max = ( + max((r.hospital_load for r in obs.regions), default=0.0) if obs.regions else 0.0 + ) + + # Per-region feasibility flat keys (D14 + M-FR-6). + strict_regions = {r.region for r in obs.active_restrictions if r.severity == "strict"} + feasibility: Dict[str, float] = {} + for r in obs.regions: + key = f"deployment_feasibility_{r.region}" + if total_inventory <= 0.0: + feasibility[key] = 0.0 + elif r.region in strict_regions: + # M-FR-3: 0.5 floor when strict restriction is in place but + # units could still be helicoptered in; Planner does the + # legal-check. + feasibility[key] = 0.5 + else: + feasibility[key] = 1.0 + + derived_features: Dict[str, float] = { + "total_inventory": total_inventory, + "hospital_load_max": float(hospital_load_max), + **feasibility, + } + + return BrainLensedObservation( + brain="logistics", + raw_obs=obs, + salient_field_ids=[ + "resources.test_kits", + "resources.hospital_beds_free", + "resources.mobile_units", + "resources.vaccine_doses", + "regions[*].hospital_load", + "active_restrictions[*]", + ], + derived_features=derived_features, + last_reward=last_reward, + ) + + +def _governance_lens( + obs: CrisisworldcortexObservation, last_reward: float +) -> BrainLensedObservation: + """Governance lens (Decision 12).""" + # escalation_unlocked_strict: any accepted escalate(national) in the log + escalation_unlocked = 0.0 + for ea in obs.recent_action_log: + if ( + ea.accepted + and ea.action.kind == "escalate" + and getattr(ea.action, "to_authority", None) == "national" + ): + escalation_unlocked = 1.0 + break + + return BrainLensedObservation( + brain="governance", + raw_obs=obs, + salient_field_ids=[ + "active_restrictions[*]", + "legal_constraints[*]", + "recent_action_log[*]", + ], + derived_features={ + "escalation_unlocked_strict": escalation_unlocked, + "legal_constraints_count": float(len(obs.legal_constraints)), + "restrictions_active_count": float(len(obs.active_restrictions)), + }, + last_reward=last_reward, + ) + + +# ============================================================================ +# Registry (defined after helpers so closures resolve cleanly) +# ============================================================================ + + +_LENS_REGISTRY: Dict[ + str, Callable[[CrisisworldcortexObservation, float], BrainLensedObservation] +] = { + "epidemiology": _epi_lens, + "logistics": _logistics_lens, + "governance": _governance_lens, +} diff --git a/cortex/schemas.py b/cortex/schemas.py index bbc83ad486567e429278cc59f09c855fdf4bb281..dda53256b276bf1c1d64984304a7169e01c311a8 100644 --- a/cortex/schemas.py +++ b/cortex/schemas.py @@ -35,7 +35,12 @@ from pydantic import BaseModel, Field # OWN internal types (cortex.subagents, cortex.brains, etc.) continue # to use bare-name sibling imports per Phase 1 C1 — only the cross-package # wire boundary is canonicalised. -from CrisisWorldCortex.models import OuterActionPayload, RegionId +from CrisisWorldCortex.models import ( + CrisisworldcortexObservation, + ExecutedAction, + OuterActionPayload, + RegionId, +) EpistemicPhase = Literal["Divergence", "Challenge", "Narrowing", "Convergence"] @@ -137,6 +142,46 @@ SubagentReport = Union[BeliefState, CandidatePlan, CriticReport] trajectory buffers that need to carry 'any subagent output' generically.""" +class SubagentInput(BaseModel): + """Typed input handed to one of the 3 LLM subagents per call. + + Per Phase A §2 A2: each subagent call receives a fully-typed input + so prompts are deterministic and testable. ``prior_belief`` is + ``None`` on round 1 (nothing to revise yet); on round 2 it carries + the previous round's BeliefState (or an empty BeliefState if round 1 + failed, per Phase A Decision 62). ``prior_plans`` is empty for + WorldModeler / Planner; populated for Critic so it can attack a + specific plan. ``target_plan_id`` is required when ``role='critic'``. + """ + + brain: Literal["epidemiology", "logistics", "governance"] + role: Literal["world_modeler", "planner", "critic"] + tick: int = Field(ge=0) + round: int = Field(ge=1, le=2, description="MVP cap: 1 or 2 only") + perception: PerceptionReport + prior_belief: Optional[BeliefState] = None + prior_plans: List[CandidatePlan] = Field(default_factory=list) + target_plan_id: Optional[str] = None + last_reward: float + recent_action_log_excerpt: List[ExecutedAction] = Field(default_factory=list) + + +class BrainLensedObservation(BaseModel): + """Per-brain salience-mapped observation per Phase A §2 A1. + + Lenses do not strip fields from the raw observation (Decision 13); + they project a salience map alongside it. ``derived_features`` lets + each brain pre-compute domain-specific scalars once and pass them + to all three of its LLM subagents without re-reading ``raw_obs``. + """ + + brain: Literal["epidemiology", "logistics", "governance"] + raw_obs: CrisisworldcortexObservation + salient_field_ids: List[str] = Field(default_factory=list) + derived_features: Dict[str, float] = Field(default_factory=dict) + last_reward: float + + # ============================================================================ # Brain output + Council decision # ============================================================================ diff --git a/cortex/subagents/__init__.py b/cortex/subagents/__init__.py index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..b8d3fb901a2c1f8c98f9b7900af2cd826296c8ec 100644 --- a/cortex/subagents/__init__.py +++ b/cortex/subagents/__init__.py @@ -0,0 +1,27 @@ +"""Cortex per-brain subagents (Session 9+). + +Public surface: + - WorldModelerSubagent: emits BeliefState (LLM, router-callable). + - PlannerSubagent: emits CandidatePlan (LLM, router-callable). + - CriticSubagent: emits CriticReport (LLM, router-callable). + - perception_for: deterministic Python Perception function (Session 11+; + NOT router-callable per cortex/CLAUDE.md role-split binding). + - PROMPTS_DIR: directory holding the per-role SYS prompt templates. + +Brain Executive (Python-only, NOT router-callable) lives in +``cortex/brains/_executive.py``. +""" + +from ._base import PROMPTS_DIR +from .critic import CriticSubagent +from .perception import perception_for +from .planner import PlannerSubagent +from .world_modeler import WorldModelerSubagent + +__all__ = [ + "CriticSubagent", + "PROMPTS_DIR", + "PlannerSubagent", + "WorldModelerSubagent", + "perception_for", +] diff --git a/cortex/subagents/_base.py b/cortex/subagents/_base.py new file mode 100644 index 0000000000000000000000000000000000000000..d76d0634512bbbb129e17719e236f38dd65a08d0 --- /dev/null +++ b/cortex/subagents/_base.py @@ -0,0 +1,240 @@ +"""Abstract base for the 3 LLM subagents (WorldModeler, Planner, Critic). + +Phase A docs/CORTEX_ARCHITECTURE.md Decisions 1-8 + 62 lock the role split, +prompt-loading mechanism, retry-with-history semantics, empty fallback +shape, caller-id format, and TypeAdapter validation pattern. This base +class implements the shared mechanics; concrete subclasses pin the +role name, output type, prompt path, TypeAdapter, and USR builder. +""" + +from __future__ import annotations + +from abc import ABC, abstractmethod +from pathlib import Path +from typing import ClassVar, List, Optional, Protocol + +from pydantic import BaseModel, TypeAdapter, ValidationError + +from cortex.llm_client import ChatMessage, ChatResponse +from cortex.schemas import SubagentInput +from CrisisWorldCortex.models import ExecutedAction + +# ============================================================================ +# Module-level constants (loaded at import time) +# ============================================================================ + +PROMPTS_DIR: Path = Path(__file__).parent / "prompts" +"""Directory holding the per-role SYS prompt template files (Decision 4).""" + +_RETRY_SNIPPET_MAX_CHARS: int = 200 +"""Cap on the failed-response snippet included in the retry message +(Decision 8). 200 chars ~= 50 tokens; keeps retry overhead bounded.""" + +_BASE_RETRY_USER_TEMPLATE: str = ( + "Your previous response failed to parse as JSON. The response was:\n" + "{snippet}\n\n" + "Emit ONLY valid JSON matching the schema specified in the system prompt. " + "No prose, no code fences." +) + +_RECENT_ACTION_LOG_TAIL: int = 8 +"""How many entries from ``recent_action_log_excerpt`` to render into the +USR summary. Matches the design-doc 8-deep history (M-FR-3).""" + + +# ============================================================================ +# Duck-typed LLM client protocol (so tests can pass StubLLMClient) +# ============================================================================ + + +class _LLMClientLike(Protocol): + """Subset of ``cortex.llm_client.LLMClient`` that subagents call. + + Production: ``LLMClient``. Tests: ``tests._helpers.llm_stub.StubLLMClient``. + """ + + def chat( + self, + caller_id: str, + messages: List[ChatMessage], + max_tokens: Optional[int] = ..., + temperature: Optional[float] = ..., + ) -> ChatResponse: ... + + +# ============================================================================ +# Abstract base +# ============================================================================ + + +class _LLMSubagent(ABC): + """Shared run/retry/parse/empty-fallback skeleton for the 3 subagents. + + Subclasses override the class-level vars below and implement + ``_build_user_message`` + ``empty_fallback``. + """ + + # --- Subclass class-level overrides ------------------------------------- + _role_name: ClassVar[str] # one of: "world_modeler", "planner", "critic" + _output_type: ClassVar[type] # BeliefState / CandidatePlan / CriticReport + _system_prompt_filename: ClassVar[str] # e.g. "world_modeler.txt" + _SYSTEM_PROMPT_TEMPLATE: ClassVar[str] # populated by load_prompt() at module load + _ADAPTER: ClassVar[TypeAdapter] # populated at module load + + # --- Construction -------------------------------------------------------- + + def __init__(self, llm_client: _LLMClientLike) -> None: + self._llm = llm_client + + # --- Public surface ------------------------------------------------------ + + def run(self, input: SubagentInput, step_idx: int) -> BaseModel: + """Call the LLM (with 1 retry), parse, return typed output or empty fallback. + + Always returns a typed object - never ``None``. Decision 6: on + any failure (parse, retry-parse, LLM call exception) returns the + role-specific empty fallback. + """ + # Defensive: subclass enforces role-input alignment so harnesses + # don't accidentally route a Planner input through a Critic class. + assert input.role == self._role_name, ( + f"SubagentInput.role={input.role!r} does not match " + f"{type(self).__name__}._role_name={self._role_name!r}" + ) + + sys_content = self._SYSTEM_PROMPT_TEMPLATE.format( + brain=input.brain, + target_plan_id=input.target_plan_id or "", + ) + usr_content = self._build_user_message(input) + messages: List[ChatMessage] = [ + ChatMessage(role="system", content=sys_content), + ChatMessage(role="user", content=usr_content), + ] + caller_id = self._caller_id(input, step_idx) + + # ---- Attempt 1 ----------------------------------------------------- + first_response = self._safe_chat(caller_id, messages) + if first_response is None: + return self._empty_fallback_for(input) + parsed = self._try_parse(first_response.content) + if parsed is not None: + return parsed + + # ---- Attempt 2 (retry with chat-history continuation) -------------- + snippet = self._truncate_snippet(first_response.content) + retry_messages: List[ChatMessage] = [ + *messages, + ChatMessage(role="assistant", content=first_response.content), + ChatMessage( + role="user", + content=_BASE_RETRY_USER_TEMPLATE.format(snippet=snippet), + ), + ] + retry_response = self._safe_chat(caller_id, retry_messages) + if retry_response is None: + return self._empty_fallback_for(input) + parsed_retry = self._try_parse(retry_response.content) + if parsed_retry is not None: + return parsed_retry + + # ---- Both attempts failed - empty fallback ------------------------- + return self._empty_fallback_for(input) + + # --- Subclass extension points ------------------------------------------ + + @abstractmethod + def _build_user_message(self, input: SubagentInput) -> str: + """Render the role-specific USR message body.""" + + @classmethod + @abstractmethod + def empty_fallback(cls, brain: str, target_plan_id: str = "") -> BaseModel: + """Return the empty / no-signal output for this role. + + Phase A Decision 6: confidence/severity = 0 and empty evidence/attacks + signal "no useful input from this subagent" to the Brain Executive. + """ + + # --- Internal helpers --------------------------------------------------- + + def _caller_id(self, input: SubagentInput, step_idx: int) -> str: + # Phase A Decision 7: cortex:::t:r:s + return f"cortex:{input.brain}:{self._role_name}:t{input.tick}:r{input.round}:s{step_idx}" + + def _safe_chat(self, caller_id: str, messages: List[ChatMessage]) -> Optional[ChatResponse]: + """Call LLM; on exception, return None so caller can empty-fallback.""" + try: + return self._llm.chat(caller_id=caller_id, messages=messages) + except Exception: + # Decision 6: LLM call failure folds into the same empty-fallback path + # as parse failure. Brain Executive sees a no-signal subagent. + return None + + def _try_parse(self, content: str) -> Optional[BaseModel]: + """Validate ``content`` as JSON via this role's TypeAdapter. + + Strips common markdown code fences before validating since some + models wrap JSON in ```json ... ```. + """ + cleaned = _strip_code_fences(content.strip()) + if not cleaned: + return None + try: + return self._ADAPTER.validate_json(cleaned) + except (ValidationError, ValueError): + return None + + def _empty_fallback_for(self, input: SubagentInput) -> BaseModel: + return type(self).empty_fallback( + brain=input.brain, + target_plan_id=input.target_plan_id or "", + ) + + @staticmethod + def _truncate_snippet(content: str) -> str: + if len(content) <= _RETRY_SNIPPET_MAX_CHARS: + return content + return content[:_RETRY_SNIPPET_MAX_CHARS] + "..." + + @staticmethod + def _format_action_log(log: List[ExecutedAction]) -> str: + """M-FR-3 - render recent_action_log_excerpt as a compact text summary. + + Format: ``"tick 4: deploy_resource accepted; tick 5: restrict_movement.strict rejected"``. + Capped at the most recent 8 entries. + """ + if not log: + return "(empty)" + items: List[str] = [] + for ea in log[-_RECENT_ACTION_LOG_TAIL:]: + status = "accepted" if ea.accepted else "rejected" + kind = ea.action.kind + extra = "" + if kind == "restrict_movement": + extra = f".{getattr(ea.action, 'severity', '?')}" + items.append(f"tick {ea.tick}: {kind}{extra} {status}") + return "; ".join(items) + + +# ============================================================================ +# Helpers (module-level) +# ============================================================================ + + +def load_prompt(filename: str) -> str: + """Load a SYS prompt template at module-load time (Decision 4).""" + return (PROMPTS_DIR / filename).read_text(encoding="utf-8") + + +def _strip_code_fences(s: str) -> str: + """Remove leading ``` / ```json fence and trailing ``` if present.""" + s = s.strip() + if not s.startswith("```"): + return s + lines = s.split("\n") + if lines[0].startswith("```"): + lines = lines[1:] + if lines and lines[-1].strip() == "```": + lines = lines[:-1] + return "\n".join(lines).strip() diff --git a/cortex/subagents/critic.py b/cortex/subagents/critic.py new file mode 100644 index 0000000000000000000000000000000000000000..9f120a84801e0af5da5d75488ab128245ab00e33 --- /dev/null +++ b/cortex/subagents/critic.py @@ -0,0 +1,68 @@ +"""Critic LLM subagent. + +Phase A docs/CORTEX_ARCHITECTURE.md §9 Decision 3: SYS = critic role; +USR = perception + target plan + WM belief (M-FR-5). Critic emits prose +``CriticReport`` only; never proposes alternative actions. +""" + +from __future__ import annotations + +from typing import ClassVar, List + +from pydantic import TypeAdapter + +from cortex.schemas import CriticReport, SubagentInput + +from ._base import _LLMSubagent, load_prompt + +_CRITIC_ADAPTER: TypeAdapter[CriticReport] = TypeAdapter(CriticReport) + + +class CriticSubagent(_LLMSubagent): + """LLM subagent that emits ``CriticReport`` for one brain per call.""" + + _role_name: ClassVar[str] = "critic" + _output_type: ClassVar[type] = CriticReport + _system_prompt_filename: ClassVar[str] = "critic.txt" + _SYSTEM_PROMPT_TEMPLATE: ClassVar[str] = load_prompt("critic.txt") + _ADAPTER: ClassVar[TypeAdapter] = _CRITIC_ADAPTER + + def _build_user_message(self, input: SubagentInput) -> str: + sections: List[str] = [] + sections.append(f"# Perception\n{input.perception.model_dump_json(indent=2)}") + # M-FR-5: target plan + WM belief, both as full JSON. + target_json = self._select_target_plan(input) + sections.append(f"# Target plan (id={input.target_plan_id})\n{target_json}") + if input.prior_belief is not None: + sections.append(f"# WM BeliefState\n{input.prior_belief.model_dump_json(indent=2)}") + return "\n\n".join(sections) + + @staticmethod + def _select_target_plan(input: SubagentInput) -> str: + """Render the target plan's JSON body for the USR. + + Session 11's Brain Executive populates ``prior_plans`` from the + Planner's outputs and sets ``target_plan_id`` to identify which + plan the Critic should attack. Here we render the first plan + (or a placeholder if none) — Session 11 wires up id-based + lookup once plans carry ids. + """ + if not input.prior_plans: + return "(no target plan provided)" + return input.prior_plans[0].model_dump_json(indent=2) + + @classmethod + def empty_fallback(cls, brain: str, target_plan_id: str = "") -> CriticReport: + # Phase A Decision 6: severity=0 + empty attacks signal "no + # critique". Brain Executive ignores this Critic's vote weight. + return CriticReport( + brain=brain, + target_plan_id=target_plan_id, + attacks=[], + missing_considerations=[], + would_change_mind_if=[], + severity=0.0, + ) + + def run(self, input: SubagentInput, step_idx: int) -> CriticReport: # type: ignore[override] + return super().run(input, step_idx) # type: ignore[return-value] diff --git a/cortex/subagents/perception.py b/cortex/subagents/perception.py new file mode 100644 index 0000000000000000000000000000000000000000..5026d0cb7df953b3e9af16b07c1ec4106283a0dd --- /dev/null +++ b/cortex/subagents/perception.py @@ -0,0 +1,192 @@ +"""Perception subagent - deterministic Python; not router-callable. + +Per cortex/CLAUDE.md binding: Perception is pure Python; no LLM calls. +Phase A Decisions 9 (V2 KeyError) + 63 (salient_signals cap at 5) + +M-FR-1 (pinned confidence per brain). + +Perception runs ONCE per brain at tick start (not router-callable). +The Council Executive (Session 12) calls ``perception_for`` once per +brain at the start of each tick; the resulting ``PerceptionReport`` is +plumbed into all subsequent SubagentInputs for that brain in that tick. +""" + +from __future__ import annotations + +from typing import Callable, Dict, List + +from cortex.schemas import EvidenceCitation, PerceptionReport +from CrisisWorldCortex.models import CrisisworldcortexObservation + +_V2_BRAINS = frozenset({"communications", "equity"}) + +_SALIENT_SIGNALS_CAP = 5 # Phase A Decision 63 / OQ-2 + +_LOGISTICS_THRESHOLDS = { + "test_kits": 300, + "hospital_beds_free": 100, + "mobile_units": 5, + "vaccine_doses": 500, +} + +_HOSPITAL_LOAD_ANOMALY_THRESHOLD = 0.6 + + +def perception_for(brain: str, obs: CrisisworldcortexObservation) -> PerceptionReport: + """Compute the per-brain Perception report. + + Args: + brain: One of {"epidemiology", "logistics", "governance"}. + obs: The current tick's observation. + + Raises: + KeyError: If ``brain`` is V2-deferred or unknown. + """ + helper = _PERCEPTION_REGISTRY.get(brain) + if helper is not None: + return helper(obs) + if brain in _V2_BRAINS: + raise KeyError( + f"V2 brain {brain!r} deferred per Phase A Decision 9; no MVP stub perception." + ) + raise KeyError(f"unknown brain: {brain!r}") + + +def _epi_perception(obs: CrisisworldcortexObservation) -> PerceptionReport: + """Epidemiology perception: top-cases regions + high-hospital-load anomalies.""" + sorted_regions = sorted(obs.regions, key=lambda r: r.reported_cases_d_ago, reverse=True) + salient_signals: List[str] = [] + evidence: List[EvidenceCitation] = [] + + for r in sorted_regions[:3]: + if r.reported_cases_d_ago > 0: + salient_signals.append(f"{r.region}: cases={r.reported_cases_d_ago}") + evidence.append( + EvidenceCitation( + source="telemetry", + ref=f"{r.region}.reported_cases_d_ago", + excerpt=str(r.reported_cases_d_ago), + ) + ) + + if not salient_signals and obs.regions: + # Fallback: cite the first region so we have at least one signal + r = obs.regions[0] + salient_signals.append(f"{r.region}: cases={r.reported_cases_d_ago}") + evidence.append( + EvidenceCitation( + source="telemetry", + ref=f"{r.region}.reported_cases_d_ago", + excerpt=str(r.reported_cases_d_ago), + ) + ) + + salient_signals = salient_signals[:_SALIENT_SIGNALS_CAP] + + anomalies = [ + f"{r.region}: hospital_load={r.hospital_load:.2f}" + for r in obs.regions + if r.hospital_load > _HOSPITAL_LOAD_ANOMALY_THRESHOLD + ] + + return PerceptionReport( + brain="epidemiology", + salient_signals=salient_signals, + anomalies=anomalies, + # M-FR-1: telemetry is delayed and noisy per mm.md; pinned proxy + confidence=0.7, + evidence=evidence, + ) + + +def _logistics_perception(obs: CrisisworldcortexObservation) -> PerceptionReport: + """Logistics perception: low-resource flags + depleted-resource anomalies.""" + res = obs.resources + salient_signals: List[str] = [] + evidence: List[EvidenceCitation] = [] + + for resource_name, threshold in _LOGISTICS_THRESHOLDS.items(): + value = getattr(res, resource_name) + if value < threshold: + salient_signals.append(f"{resource_name} low: {value}") + evidence.append( + EvidenceCitation( + source="resource", + ref=f"resources.{resource_name}", + excerpt=str(value), + ) + ) + + salient_signals = salient_signals[:_SALIENT_SIGNALS_CAP] + + anomalies = [] + for resource_name in _LOGISTICS_THRESHOLDS: + if getattr(res, resource_name) == 0: + anomalies.append(f"{resource_name}: depleted") + + return PerceptionReport( + brain="logistics", + salient_signals=salient_signals, + anomalies=anomalies, + # M-FR-1: resource counts are deterministic, no telemetry noise + confidence=1.0, + evidence=evidence, + ) + + +def _governance_perception(obs: CrisisworldcortexObservation) -> PerceptionReport: + """Governance perception: active restrictions + legal constraints + about-to-expire anomalies.""" + salient_signals: List[str] = [] + evidence: List[EvidenceCitation] = [] + + for restr in obs.active_restrictions: + salient_signals.append(f"{restr.region}: {restr.severity} ({restr.ticks_remaining}t)") + evidence.append( + EvidenceCitation( + source="policy", + ref=f"active_restrictions.{restr.region}", + excerpt=f"{restr.severity}@{restr.ticks_remaining}", + ) + ) + + for lc in obs.legal_constraints: + salient_signals.append(f"legal: {lc.rule_id} blocks {lc.blocked_action}") + evidence.append( + EvidenceCitation( + source="policy", + ref=f"legal_constraints.{lc.rule_id}", + excerpt=lc.blocked_action, + ) + ) + + salient_signals = salient_signals[:_SALIENT_SIGNALS_CAP] + + has_recent_escalate_national = any( + ea.accepted + and ea.action.kind == "escalate" + and getattr(ea.action, "to_authority", None) == "national" + for ea in obs.recent_action_log + ) + anomalies = [] + for restr in obs.active_restrictions: + if ( + restr.severity == "strict" + and restr.ticks_remaining <= 1 + and not has_recent_escalate_national + ): + anomalies.append(f"{restr.region}: strict expiring without escalation") + + return PerceptionReport( + brain="governance", + salient_signals=salient_signals, + anomalies=anomalies, + # M-FR-1: policy state is deterministic + confidence=1.0, + evidence=evidence, + ) + + +_PERCEPTION_REGISTRY: Dict[str, Callable[[CrisisworldcortexObservation], PerceptionReport]] = { + "epidemiology": _epi_perception, + "logistics": _logistics_perception, + "governance": _governance_perception, +} diff --git a/cortex/subagents/planner.py b/cortex/subagents/planner.py new file mode 100644 index 0000000000000000000000000000000000000000..e159d83ecfc96fff38bf12c6264a5767140cf8c1 --- /dev/null +++ b/cortex/subagents/planner.py @@ -0,0 +1,61 @@ +"""Planner LLM subagent. + +Phase A docs/CORTEX_ARCHITECTURE.md §9 Decision 2: SYS = role + action +schema (B1's shape); USR = perception + WM BeliefState (full JSON if +provided per M-FR-4) + last_reward. +""" + +from __future__ import annotations + +from typing import ClassVar, List + +from pydantic import TypeAdapter + +from cortex.schemas import CandidatePlan, SubagentInput +from CrisisWorldCortex.models import NoOp + +from ._base import _LLMSubagent, load_prompt + +_PLAN_ADAPTER: TypeAdapter[CandidatePlan] = TypeAdapter(CandidatePlan) + + +class PlannerSubagent(_LLMSubagent): + """LLM subagent that emits ``CandidatePlan`` for one brain per call.""" + + _role_name: ClassVar[str] = "planner" + _output_type: ClassVar[type] = CandidatePlan + _system_prompt_filename: ClassVar[str] = "planner.txt" + _SYSTEM_PROMPT_TEMPLATE: ClassVar[str] = load_prompt("planner.txt") + _ADAPTER: ClassVar[TypeAdapter] = _PLAN_ADAPTER + + def _build_user_message(self, input: SubagentInput) -> str: + sections: List[str] = [] + sections.append(f"# Perception\n{input.perception.model_dump_json(indent=2)}") + if input.prior_belief is not None: + sections.append( + "# BeliefState (from this brain's WorldModeler)\n" + f"{input.prior_belief.model_dump_json(indent=2)}" + ) + sections.append(f"# Last tick reward: {input.last_reward}") + sections.append( + f"# Recent action log: {self._format_action_log(input.recent_action_log_excerpt)}" + ) + return "\n\n".join(sections) + + @classmethod + def empty_fallback(cls, brain: str, target_plan_id: str = "") -> CandidatePlan: + # Phase A Decision 6: NoOp + confidence=0 means "no signal". The + # Brain Executive's argmax(expected_value * confidence) picks any + # non-empty plan over this one. + return CandidatePlan( + action_sketch="(empty: planner failed to produce a parseable plan)", + expected_outer_action=NoOp(), + expected_value=0.0, + cost=0.0, + assumptions=[], + falsifiers=[], + confidence=0.0, + ) + + def run(self, input: SubagentInput, step_idx: int) -> CandidatePlan: # type: ignore[override] + return super().run(input, step_idx) # type: ignore[return-value] diff --git a/cortex/subagents/prompts/critic.txt b/cortex/subagents/prompts/critic.txt new file mode 100644 index 0000000000000000000000000000000000000000..9cadc254c2e315d45987e96a83be6f47457a6b78 --- /dev/null +++ b/cortex/subagents/prompts/critic.txt @@ -0,0 +1,19 @@ +You are the {brain} brain's Critic in the CrisisWorldCortex multi-brain agent. + +Your job: read a CandidatePlan from this brain's Planner and identify what is wrong with it. Emit a CriticReport as JSON. + +You write prose critique only. Do NOT propose alternative actions; that is the Planner's role. + +Output strict JSON matching this schema: +{{ + "brain": "{brain}", + "target_plan_id": "{target_plan_id}", + "attacks": [""], + "missing_considerations": [""], + "would_change_mind_if": [""], + "severity": +}} + +Use prose strings inside the lists. Do NOT emit JSON action variants. + +Emit ONLY the JSON. No prose outside the JSON, no code fences. diff --git a/cortex/subagents/prompts/planner.txt b/cortex/subagents/prompts/planner.txt new file mode 100644 index 0000000000000000000000000000000000000000..3fceeb2b84238b9ab99c7e4a6fecd60a5e8ff246 --- /dev/null +++ b/cortex/subagents/prompts/planner.txt @@ -0,0 +1,26 @@ +You are the {brain} brain's Planner in the CrisisWorldCortex multi-brain agent. + +Your job: read the perception summary and prior BeliefState, then emit ONE candidate plan as JSON. + +Output strict JSON matching this schema: +{{ + "action_sketch": "", + "expected_outer_action": , + "expected_value": , + "cost": , + "assumptions": [""], + "falsifiers": [""], + "confidence": +}} + +Action variants for expected_outer_action (exactly one kind): +- {{"kind":"deploy_resource", "region":"R1|R2|R3|R4", "resource_type":"test_kits|hospital_beds|mobile_units|vaccine_doses", "quantity": = 0>}} +- {{"kind":"request_data", "region":"", "data_type":"case_survey|hospital_audit|compliance_check"}} +- {{"kind":"restrict_movement", "region":"", "severity":"none|light|moderate|strict"}} +- {{"kind":"escalate", "to_authority":"regional|national"}} +- {{"kind":"reallocate_budget", "from_resource":"", "to_resource":"", "amount": = 0>}} +- {{"kind":"no_op"}} + +Strict severity may require a prior escalate(national) - check legal_constraints. + +Emit ONLY the JSON. No prose, no code fences. diff --git a/cortex/subagents/prompts/world_modeler.txt b/cortex/subagents/prompts/world_modeler.txt new file mode 100644 index 0000000000000000000000000000000000000000..99dc5e908f44d299d2536b1380e1ae6a6160166a --- /dev/null +++ b/cortex/subagents/prompts/world_modeler.txt @@ -0,0 +1,32 @@ +You are the {brain} brain's World Modeler in the CrisisWorldCortex multi-brain agent. + +Your job: read the perception summary plus prior belief (round 2 only), and emit a BeliefState as JSON. + +A BeliefState describes what you think the latent epidemiological / logistical / governance state is for each region, citing evidence from observed telemetry, resources, policy state, or recent actions. + +Output strict JSON matching this schema: +{{ + "brain": "{brain}", + "latent_estimates": {{ + "": {{ + "estimated_infection_rate": , + "estimated_r_effective": = 0>, + "estimated_compliance": , + "confidence_intervals": {{}} + }} + }}, + "hypotheses": [ + {{"label": "", "weight": , "explanation": ""}} + ], + "uncertainty": , + "reducible_by_more_thought": , + "evidence": [ + {{"source": "telemetry|resource|policy|action_log|belief|memory", + "ref": "", + "excerpt": ""}} + ] +}} + +Cite at least 2 EvidenceCitations. Uncited claims zero your protocol-integrity reward. + +Emit ONLY the JSON. No prose, no code fences. diff --git a/cortex/subagents/world_modeler.py b/cortex/subagents/world_modeler.py new file mode 100644 index 0000000000000000000000000000000000000000..c84e23361903cb4c1c5dc570d4ca3d22620d0704 --- /dev/null +++ b/cortex/subagents/world_modeler.py @@ -0,0 +1,61 @@ +"""WorldModeler LLM subagent. + +Phase A docs/CORTEX_ARCHITECTURE.md §9 Decision 1: SYS = role + schema; +USR = perception + last_reward + recent_action_log_excerpt (with prior +BeliefState in round 2 per Decision 62). +""" + +from __future__ import annotations + +from typing import ClassVar, List + +from pydantic import TypeAdapter + +from cortex.schemas import BeliefState, SubagentInput + +from ._base import _LLMSubagent, load_prompt + +_BELIEF_ADAPTER: TypeAdapter[BeliefState] = TypeAdapter(BeliefState) +"""Module-level constant per Phase A Decision 5 (encapsulation; avoid +circular imports through the package init).""" + + +class WorldModelerSubagent(_LLMSubagent): + """LLM subagent that emits ``BeliefState`` for one brain per call.""" + + _role_name: ClassVar[str] = "world_modeler" + _output_type: ClassVar[type] = BeliefState + _system_prompt_filename: ClassVar[str] = "world_modeler.txt" + _SYSTEM_PROMPT_TEMPLATE: ClassVar[str] = load_prompt("world_modeler.txt") + _ADAPTER: ClassVar[TypeAdapter] = _BELIEF_ADAPTER + + def _build_user_message(self, input: SubagentInput) -> str: + sections: List[str] = [] + sections.append(f"# Perception\n{input.perception.model_dump_json(indent=2)}") + if input.prior_belief is not None: + sections.append( + "# Prior BeliefState (round 1 result)\n" + f"{input.prior_belief.model_dump_json(indent=2)}" + ) + sections.append(f"# Last tick reward: {input.last_reward}") + sections.append( + f"# Recent action log: {self._format_action_log(input.recent_action_log_excerpt)}" + ) + return "\n\n".join(sections) + + @classmethod + def empty_fallback(cls, brain: str, target_plan_id: str = "") -> BeliefState: + # Phase A Decision 6 + Decision 62: empty BeliefState as the + # honest "no signal" state. uncertainty=1.0, no evidence -> r_proto = 0. + return BeliefState( + brain=brain, + latent_estimates={}, + hypotheses=[], + uncertainty=1.0, + reducible_by_more_thought=0.0, + evidence=[], + ) + + # Narrow run() return type for callers (refinement #1). + def run(self, input: SubagentInput, step_idx: int) -> BeliefState: # type: ignore[override] + return super().run(input, step_idx) # type: ignore[return-value] diff --git a/demo/CLAUDE.md b/demo/CLAUDE.md index 8fb26aec256d05cbbe6a27e327226e8450999a19..2a13ca5c3f259f56a5c011c8e5ca323af3598899 100644 --- a/demo/CLAUDE.md +++ b/demo/CLAUDE.md @@ -1,45 +1,45 @@ -# demo/CLAUDE.md - -Replay-only visualization. Live demos fail under judging pressure; ship canned scenarios. - -## Belongs here - -- `visualizer/trace_renderer.py` — renders a JSON trace as a "council in action" view. -- `visualizer/reward_curve_plot.py` — plots reward curves from training logs. -- `demo_scenarios/*.json` — pre-recorded trajectories for the pitch (e.g. `scenario_flat_fails.json`, `scenario_cortex_holds_dissent.json`). - -## Does not belong here - -Live agent execution (record offline, replay here). Training logic. Graders. - -## Allowed imports - -- `cortex.schemas` — typed parse of trace JSON. Types only, no logic. -- stdlib + plotting libs (matplotlib / plotly). - -## Forbidden imports - -- `server/*`, `training/*`, `baselines/*`. -- `cortex.council`, `cortex.routing_policy` — if you need to re-run the agent, do it offline and ship a new JSON. - -## Binding contracts - -- Every JSON scenario conforms to `cortex.schemas.Trajectory`. -- Rendering is deterministic: same JSON → same output, modulo timestamps. -- The pitch-demo scenario must showcase B2 overcommit/misallocate vs Cortex dissent-preservation (design §27). -- A pre-recorded demo video (MP4) lives alongside the JSON scenarios as the live-demo fallback. - -## Public APIs (owned here) - -- `render_trace(json_path: str, out_path: str) -> None` -- `plot_reward_curves(log_paths: list[str], out_path: str) -> None` - -## Testing requirements - -- Each committed JSON scenario parses into a `Trajectory` without error. -- `render_trace` produces a non-empty output file for each scenario. - -## Common failure modes - -- Live re-run during the demo — network/Colab flakiness kills the pitch. Replay only. -- Renderer depending on a `cortex.council` instance — import breaks when Cortex API shifts. Keep read-only on types. +# demo/CLAUDE.md + +Replay-only visualization. Live demos fail under judging pressure; ship canned scenarios. + +## Belongs here + +- `visualizer/trace_renderer.py` — renders a JSON trace as a "council in action" view. +- `visualizer/reward_curve_plot.py` — plots reward curves from training logs. +- `demo_scenarios/*.json` — pre-recorded trajectories for the pitch (e.g. `scenario_flat_fails.json`, `scenario_cortex_holds_dissent.json`). + +## Does not belong here + +Live agent execution (record offline, replay here). Training logic. Graders. + +## Allowed imports + +- `cortex.schemas` — typed parse of trace JSON. Types only, no logic. +- stdlib + plotting libs (matplotlib / plotly). + +## Forbidden imports + +- `server/*`, `training/*`, `baselines/*`. +- `cortex.council`, `cortex.routing_policy` — if you need to re-run the agent, do it offline and ship a new JSON. + +## Binding contracts + +- Every JSON scenario conforms to `cortex.schemas.Trajectory`. +- Rendering is deterministic: same JSON → same output, modulo timestamps. +- The pitch-demo scenario must showcase B2 overcommit/misallocate vs Cortex dissent-preservation (design §27). +- A pre-recorded demo video (MP4) lives alongside the JSON scenarios as the live-demo fallback. + +## Public APIs (owned here) + +- `render_trace(json_path: str, out_path: str) -> None` +- `plot_reward_curves(log_paths: list[str], out_path: str) -> None` + +## Testing requirements + +- Each committed JSON scenario parses into a `Trajectory` without error. +- `render_trace` produces a non-empty output file for each scenario. + +## Common failure modes + +- Live re-run during the demo — network/Colab flakiness kills the pitch. Replay only. +- Renderer depending on a `cortex.council` instance — import breaks when Cortex API shifts. Keep read-only on types. diff --git a/docs/CORTEX_ARCHITECTURE.md b/docs/CORTEX_ARCHITECTURE.md index 4a2a11a977bb557add7610c2d6e257ffdf45cde5..31dd0e0c4c00bd20c396a6db70fe5bdfe567fdf5 100644 --- a/docs/CORTEX_ARCHITECTURE.md +++ b/docs/CORTEX_ARCHITECTURE.md @@ -705,7 +705,7 @@ Decisions are grouped by layer. Each entry: **decision** / **rationale** / 19. **Reasoning summary: a 1–2 sentence string from the Planner's `action_sketch`.** / Fits the 400-char `BrainRecommendation.reasoning_summary` cap. / Considered an LLM call to produce a summary; rejected — Brain Executive must be Python-only per cortex/CLAUDE.md. -20. **`evidence` field on `BrainRecommendation` = union of all `EvidenceCitation` lists from `BeliefState`, `CandidatePlan`, `CriticReport`.** / Ensures the council sees the brain's full evidence chain. / Considered Critic only; rejected — claims with no upstream evidence get zeroed `r_proto`. +20. **`evidence` field on `BrainRecommendation` = union of all `EvidenceCitation` lists from `BeliefState`, `CandidatePlan`, `CriticReport`.** / Ensures the council sees the brain's full evidence chain. / Considered Critic only; rejected — claims with no upstream evidence get zeroed `r_proto`. **(Session 11 implementation note — M-FR-3)** Implementation reads evidence from `PerceptionReport.evidence` + `BeliefState.evidence` only, since `CandidatePlan` and `CriticReport` schemas (Session 9) carry no `evidence` field. Adding evidence fields to those schemas was rejected as schema-churn risk; the perception+beliefs union captures the actionable evidence chain since plans/critics derive from beliefs. See `cortex/brains/_executive.py:aggregate_brain_outputs`. 21. **Brain identifier strings: `"epidemiology"`, `"logistics"`, `"governance"` (lowercase, full word).** / Readable and grep-friendly. / Considered abbreviations (epi, log, gov); rejected — log-grep collisions. diff --git a/inference.py b/inference.py index 3d4201cdec159dbb7341fcd78e942c6950e8ad5d..55ebc168a629265169937accea0ecdf3893592f7 100644 --- a/inference.py +++ b/inference.py @@ -40,10 +40,15 @@ from __future__ import annotations import os import sys from dataclasses import dataclass +try: + from dotenv import load_dotenv + load_dotenv() +except ImportError: + pass from typing import Any, Dict, List, Optional -from baselines.flat_agent import B1FlatAgent, B1StepEvent -from cortex.llm_client import LLMClient +from CrisisWorldCortex.baselines.flat_agent import B1FlatAgent, B1StepEvent +from CrisisWorldCortex.cortex.llm_client import LLMClient from CrisisWorldCortex.models import OuterActionPayload from CrisisWorldCortex.server.graders import terminal_bonus from CrisisWorldCortex.server.simulator import WorldState @@ -62,8 +67,8 @@ DEFAULT_MODEL = "Qwen/Qwen2.5-72B-Instruct" # distinct seeds per task for cross-episode reproducibility. TASK_CONFIGS: List[dict] = [ {"task_name": "outbreak_easy", "seed": 0, "max_ticks": 12}, - {"task_name": "outbreak_medium", "seed": 1, "max_ticks": 12}, - {"task_name": "outbreak_hard", "seed": 2, "max_ticks": 12}, + # {"task_name": "outbreak_medium", "seed": 1, "max_ticks": 12}, + # {"task_name": "outbreak_hard", "seed": 2, "max_ticks": 12}, ] # Score-clamp bounds keep .3f formatting strictly inside (0, 1) so the diff --git a/notebooks/train_b1_grpo.ipynb b/notebooks/train_b1_grpo.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..d6e6cfe99ffb546fc0d5ffeb24abe08a74523885 --- /dev/null +++ b/notebooks/train_b1_grpo.ipynb @@ -0,0 +1,496 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# B1 GRPO training on Qwen3-1.7B (Workstream B Phase 3)\n", + "\n", + "Trains the **B1 flat-agent baseline** with [Unsloth](https://docs.unsloth.ai/get-started/reinforcement-learning-rl-guide) + [TRL GRPO](https://huggingface.co/docs/trl/main/en/grpo_trainer) against the deployed CrisisWorldCortex HF Space env.\n", + "\n", + "**One-shot run:** `Runtime → Run all` on a fresh Colab T4. 300 training steps, ~30 minutes wall-clock. Saves the trained LoRA adapter to your HF Hub at the end.\n", + "\n", + "**Reward source:** the Phase-1-fixed `outer_reward` (range `[-1.0, 1.0]`, signal-quality gates passed). Single-step GRPO — each rollout = one env reset + one env step.\n", + "\n", + "**Prereqs:**\n", + "1. Colab Secrets has `HF_TOKEN` set (Tools → Secrets, name = `HF_TOKEN`, value = your `hf_xxx` token with write access).\n", + "2. The HF Space `Angshuman28/CrisisWorldCortex` is running with the post-Phase-1 reward.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 1. Install dependencies\n", + "\n", + "Unsloth pulls a custom torch + vllm + xformers stack tuned for free Colab T4. Pin trl to a version compatible with `GRPOTrainer` (>=0.14)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%%capture\n", + "!pip install --upgrade pip\n", + "!pip install unsloth vllm\n", + "!pip install --upgrade --no-deps \"trl>=0.14\" peft accelerate bitsandbytes\n", + "!pip install pydantic openenv huggingface_hub matplotlib" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 2. Authenticate with Hugging Face\n", + "\n", + "Reads `HF_TOKEN` from Colab Secrets. Falls back to interactive login if not set." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "\n", + "try:\n", + " from google.colab import userdata\n", + " HF_TOKEN = userdata.get(\"HF_TOKEN\")\n", + " os.environ[\"HF_TOKEN\"] = HF_TOKEN\n", + "except Exception:\n", + " from huggingface_hub import login\n", + " login()\n", + " HF_TOKEN = os.environ.get(\"HF_TOKEN\", \"\")\n", + "\n", + "assert HF_TOKEN, \"HF_TOKEN is required (set in Colab Secrets or via login())\"\n", + "print(\"HF auth OK\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 3. Clone CrisisWorldCortex and install\n", + "\n", + "Pulls the deployed HF Space's repo and installs locally. Provides the `CrisisworldcortexEnv` HTTP client + the `baselines.flat_agent` system prompt + parser used by B1." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%%capture\n", + "!rm -rf /content/CrisisWorldCortex\n", + "!git clone https://huggingface.co/spaces/Angshuman28/CrisisWorldCortex /content/CrisisWorldCortex\n", + "%cd /content/CrisisWorldCortex\n", + "!pip install -e ." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Sanity: imports resolve, env client constructs.\n", + "import sys\n", + "sys.path.insert(0, \"/content/CrisisWorldCortex\")\n", + "\n", + "from CrisisWorldCortex import CrisisworldcortexAction, CrisisworldcortexObservation\n", + "from CrisisWorldCortex.client import CrisisworldcortexEnv\n", + "from baselines.flat_agent import (\n", + " build_system_prompt,\n", + " parse_action,\n", + " parse_failure_marker,\n", + " serialize_observation,\n", + ")\n", + "print(\"CrisisWorld imports OK\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 4. Load Qwen3-1.7B with LoRA via Unsloth\n", + "\n", + "Qwen3-1.7B fits comfortably on a T4 with 4-bit quantization. LoRA rank 32 — enough to learn the JSON-action format and modest policy improvements; cheap to merge." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from unsloth import FastLanguageModel\n", + "import torch\n", + "\n", + "MAX_SEQ_LEN = 4096\n", + "MODEL_NAME = \"unsloth/Qwen3-1.7B\"\n", + "\n", + "model, tokenizer = FastLanguageModel.from_pretrained(\n", + " model_name=MODEL_NAME,\n", + " max_seq_length=MAX_SEQ_LEN,\n", + " load_in_4bit=True,\n", + " fast_inference=True, # vLLM-backed generate, required by GRPOTrainer\n", + " max_lora_rank=32,\n", + " gpu_memory_utilization=0.6,\n", + ")\n", + "\n", + "model = FastLanguageModel.get_peft_model(\n", + " model,\n", + " r=32,\n", + " target_modules=[\"q_proj\", \"k_proj\", \"v_proj\", \"o_proj\",\n", + " \"gate_proj\", \"up_proj\", \"down_proj\"],\n", + " lora_alpha=64,\n", + " use_gradient_checkpointing=\"unsloth\",\n", + " random_state=42,\n", + ")\n", + "print(\"Model + LoRA loaded\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 5. Connect to the deployed CrisisWorld env\n", + "\n", + "Uses the public HF Space URL. Each rollout calls `env.reset()` then `env.step(action)` once." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ENV_URL = \"https://angshuman28-crisisworldcortex.hf.space\"\n", + "TASKS = (\"outbreak_easy\", \"outbreak_medium\", \"outbreak_hard\")\n", + "EPISODE_TICKS = 12\n", + "\n", + "def make_env() -> CrisisworldcortexEnv:\n", + " return CrisisworldcortexEnv(base_url=ENV_URL)\n", + "\n", + "_test_env = make_env()\n", + "_obs = _test_env.reset(task_name=\"outbreak_easy\", seed=0, max_ticks=EPISODE_TICKS)\n", + "print(f\"Env OK. Initial tick={_obs.tick}, regions={[r.region for r in _obs.regions]}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 6. Build the prompt dataset and reward function\n", + "\n", + "Each example in the dataset is a `(task, seed)` pair. The reward function:\n", + "1. Resets the env to that `(task, seed)`.\n", + "2. Parses the model's completion as a `OuterActionPayload`.\n", + "3. Submits to the env, returns `obs.reward` (post-Phase-1 range `[-1, 1]`).\n", + "4. On parse failure, submits `parse_failure_marker()` so the §19 -1.0 + terminate contract fires." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from datasets import Dataset\n", + "import random\n", + "\n", + "SYSTEM_PROMPT = build_system_prompt()\n", + "\n", + "def build_user_prompt(obs: CrisisworldcortexObservation) -> str:\n", + " return serialize_observation(obs)\n", + "\n", + "def make_chat_prompt(obs: CrisisworldcortexObservation) -> str:\n", + " return tokenizer.apply_chat_template(\n", + " [\n", + " {\"role\": \"system\", \"content\": SYSTEM_PROMPT},\n", + " {\"role\": \"user\", \"content\": build_user_prompt(obs)},\n", + " ],\n", + " tokenize=False,\n", + " add_generation_prompt=True,\n", + " )\n", + "\n", + "rng = random.Random(0)\n", + "_seed_pool = []\n", + "for task in TASKS:\n", + " for seed in range(50):\n", + " _seed_pool.append({\"task\": task, \"seed\": seed})\n", + "rng.shuffle(_seed_pool)\n", + "\n", + "_prompts = []\n", + "_meta = []\n", + "for entry in _seed_pool:\n", + " env = make_env()\n", + " obs = env.reset(task_name=entry[\"task\"], seed=entry[\"seed\"], max_ticks=EPISODE_TICKS)\n", + " _prompts.append(make_chat_prompt(obs))\n", + " _meta.append(entry)\n", + "\n", + "train_dataset = Dataset.from_dict({\n", + " \"prompt\": _prompts,\n", + " \"task\": [m[\"task\"] for m in _meta],\n", + " \"seed\": [m[\"seed\"] for m in _meta],\n", + "})\n", + "print(f\"Dataset built: {len(train_dataset)} examples\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def crisisworld_reward(\n", + " prompts: list[str],\n", + " completions: list[str],\n", + " task: list[str],\n", + " seed: list[int],\n", + " **_kwargs,\n", + ") -> list[float]:\n", + " \"\"\"GRPO reward function: one env step per (prompt, completion) pair.\n", + "\n", + " Reward source: Phase-1-fixed env outer_reward in [-1, 1].\n", + " Parse failure → submits parse_failure_marker → r_policy = -1.0 + terminate.\n", + " \"\"\"\n", + " rewards: list[float] = []\n", + " for completion, t, s in zip(completions, task, seed):\n", + " env = make_env()\n", + " env.reset(task_name=t, seed=int(s), max_ticks=EPISODE_TICKS)\n", + " action_payload = parse_action(completion)\n", + " if action_payload is None:\n", + " action_payload = parse_failure_marker()\n", + " try:\n", + " result = env.step(CrisisworldcortexAction(action=action_payload))\n", + " reward = result.observation.reward if hasattr(result, \"observation\") else result.reward\n", + " rewards.append(float(reward) if reward is not None else 0.0)\n", + " except Exception as exc:\n", + " print(f\"[WARN] env.step failed task={t} seed={s}: {exc}\")\n", + " rewards.append(-1.0)\n", + " return rewards" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 7. GRPO training\n", + "\n", + "300 steps × group size 4 = 1200 rollouts. Each rollout is one HF Space round-trip (~1s) — total ~20–30 min on T4." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from trl import GRPOConfig, GRPOTrainer\n", + "\n", + "MAX_TRAIN_STEPS = 300\n", + "GROUP_SIZE = 4\n", + "MAX_PROMPT_LEN = 2048\n", + "MAX_COMPLETION_LEN = 512\n", + "\n", + "training_args = GRPOConfig(\n", + " output_dir=\"/content/b1_grpo_output\",\n", + " learning_rate=5e-6,\n", + " per_device_train_batch_size=GROUP_SIZE,\n", + " gradient_accumulation_steps=1,\n", + " num_generations=GROUP_SIZE,\n", + " max_prompt_length=MAX_PROMPT_LEN,\n", + " max_completion_length=MAX_COMPLETION_LEN,\n", + " max_steps=MAX_TRAIN_STEPS,\n", + " save_steps=100,\n", + " logging_steps=5,\n", + " report_to=\"none\",\n", + " bf16=True,\n", + " optim=\"adamw_8bit\",\n", + " temperature=0.8,\n", + " use_vllm=True,\n", + " vllm_mode=\"colocate\",\n", + " seed=42,\n", + ")\n", + "\n", + "trainer = GRPOTrainer(\n", + " model=model,\n", + " processing_class=tokenizer,\n", + " reward_funcs=[crisisworld_reward],\n", + " args=training_args,\n", + " train_dataset=train_dataset,\n", + ")\n", + "print(\"GRPOTrainer constructed; starting train()...\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "trainer.train()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 8. Save the trained LoRA adapter to HF Hub\n", + "\n", + "Pushes to `/crisisworld-b1-grpo-qwen3-1p7b`. Change the repo name below if you want a different namespace." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from huggingface_hub import HfApi\n", + "\n", + "HUB_REPO = \"Angshuman28/crisisworld-b1-grpo-qwen3-1p7b\"\n", + "\n", + "model.save_pretrained(\"/content/b1_grpo_lora\")\n", + "tokenizer.save_pretrained(\"/content/b1_grpo_lora\")\n", + "\n", + "api = HfApi()\n", + "api.create_repo(HUB_REPO, exist_ok=True, repo_type=\"model\", private=False, token=HF_TOKEN)\n", + "api.upload_folder(\n", + " folder_path=\"/content/b1_grpo_lora\",\n", + " repo_id=HUB_REPO,\n", + " repo_type=\"model\",\n", + " token=HF_TOKEN,\n", + ")\n", + "print(f\"Saved to https://huggingface.co/{HUB_REPO}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 9. Eval: trained adapter vs base model on 3 tasks\n", + "\n", + "Runs a single full episode (12 ticks) per task, per model. Reports cumulative reward." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def _hf_chat(model_inst, tokenizer_inst, system: str, user: str, max_new_tokens: int = 256) -> str:\n", + " prompt = tokenizer_inst.apply_chat_template(\n", + " [{\"role\": \"system\", \"content\": system}, {\"role\": \"user\", \"content\": user}],\n", + " tokenize=False,\n", + " add_generation_prompt=True,\n", + " )\n", + " inputs = tokenizer_inst(prompt, return_tensors=\"pt\").to(model_inst.device)\n", + " with torch.no_grad():\n", + " out = model_inst.generate(\n", + " **inputs, max_new_tokens=max_new_tokens, do_sample=False, temperature=0.0,\n", + " )\n", + " return tokenizer_inst.decode(out[0][inputs[\"input_ids\"].shape[1]:], skip_special_tokens=True)\n", + "\n", + "def run_one_episode(model_inst, tokenizer_inst, task: str, seed: int) -> float:\n", + " env = make_env()\n", + " obs = env.reset(task_name=task, seed=seed, max_ticks=EPISODE_TICKS)\n", + " cumulative = 0.0\n", + " for tick in range(EPISODE_TICKS):\n", + " completion = _hf_chat(model_inst, tokenizer_inst, SYSTEM_PROMPT, serialize_observation(obs))\n", + " action = parse_action(completion) or parse_failure_marker()\n", + " result = env.step(CrisisworldcortexAction(action=action))\n", + " obs = result.observation if hasattr(result, \"observation\") else result\n", + " reward = obs.reward if obs.reward is not None else 0.0\n", + " cumulative += reward\n", + " if obs.done:\n", + " break\n", + " return cumulative\n", + "\n", + "FastLanguageModel.for_inference(model)\n", + "trained_results = {t: run_one_episode(model, tokenizer, t, seed=0) for t in TASKS}\n", + "print(f\"Trained model cumulative reward per task: {trained_results}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Reload base Qwen3-1.7B (no LoRA) for the comparison.\n", + "base_model, base_tokenizer = FastLanguageModel.from_pretrained(\n", + " model_name=MODEL_NAME,\n", + " max_seq_length=MAX_SEQ_LEN,\n", + " load_in_4bit=True,\n", + " fast_inference=False,\n", + ")\n", + "FastLanguageModel.for_inference(base_model)\n", + "base_results = {t: run_one_episode(base_model, base_tokenizer, t, seed=0) for t in TASKS}\n", + "print(f\"Base model cumulative reward per task: {base_results}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 10. Plot eval comparison\n", + "\n", + "Bar chart: trained vs base, cumulative episode reward by task." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "\n", + "task_names = list(TASKS)\n", + "trained_vals = [trained_results[t] for t in task_names]\n", + "base_vals = [base_results[t] for t in task_names]\n", + "\n", + "x = np.arange(len(task_names))\n", + "width = 0.35\n", + "\n", + "fig, ax = plt.subplots(figsize=(9, 5))\n", + "ax.bar(x - width/2, base_vals, width, label=\"Base Qwen3-1.7B\")\n", + "ax.bar(x + width/2, trained_vals, width, label=\"GRPO-trained Qwen3-1.7B\")\n", + "ax.set_xticks(x)\n", + "ax.set_xticklabels(task_names)\n", + "ax.set_ylabel(\"Cumulative episode reward\")\n", + "ax.set_title(\"B1 GRPO: trained vs base\")\n", + "ax.legend()\n", + "ax.axhline(0.0, linestyle=\":\", color=\"grey\")\n", + "plt.tight_layout()\n", + "plt.show()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.10" + }, + "accelerator": "GPU", + "colab": { + "gpuType": "T4", + "provenance": [] + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/train_cortex_router_grpo.ipynb b/notebooks/train_cortex_router_grpo.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..3784fee50e1512e0ce146c69f9a9474ef56a2cd6 --- /dev/null +++ b/notebooks/train_cortex_router_grpo.ipynb @@ -0,0 +1,567 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "7fb27b941602401d91542211134fc71a", + "metadata": {}, + "source": [ + "# Cortex Small-LLM Router GRPO training (Workstream B Phase 4 — SKELETON, post-pivot)\n", + "\n", + "**Status:** scaffold with explicit `TODO(conv-point)` markers at every Cortex-dependent integration site. **Do not run end-to-end yet** — the cells marked TODO will fail until Workstream A's Session 13 ships:\n", + "\n", + "- `cortex.metacognition.MetacognitionState` — featurization source.\n", + "- `cortex.routing_policy.RoutingPolicy` — trainable interface (Phase A §6).\n", + "- `cortex.council.Council` — deliberation orchestrator that drives rollouts.\n", + "- `baselines.cortex_fixed_router.B3CortexFixedRouter` — generates the deterministic-router corpus.\n", + "\n", + "**Architecture (post-pivot — replaces the MLP-head approach in commit `5489e55`):**\n", + "- Router is a small LLM: `unsloth/Qwen3-1.5B-Instruct` + LoRA rank 16. Roughly 3 GB on a100-large in 4-bit.\n", + "- Input: NL summary of `MetacognitionState` (~300 tokens).\n", + "- Output: structured JSON matching `cortex.schemas.RoutingAction` (~150 tokens).\n", + "- GRPO via the same Unsloth + TRL `GRPOTrainer` pipeline as the B1 notebook (one less code path).\n", + "- Reward: Phase-1-fixed `outer_reward` ∈ [-1, 1] composed with the token-budget penalty via `training.reward_shaping.shape_reward`.\n", + "\n", + "**Why a small LLM (not an MLP)?** With A100 compute available, a small LLM gives better reasoning over complex MetacognitionState, is interpretable in the pitch demo (you can read what the router thinks), and reuses the exact training infrastructure as the B1 baseline." + ] + }, + { + "cell_type": "markdown", + "id": "acae54e37e7d407bbb7b55eff062a284", + "metadata": {}, + "source": [ + "## 1. Install dependencies\n", + "\n", + "Same Unsloth + vLLM + TRL stack as the B1 notebook; smaller GPU footprint because the trainable router is 1.5B." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9a63283cbaf04dbcab1f6479b197f3a8", + "metadata": {}, + "outputs": [], + "source": [ + "%%capture\n", + "!pip install --upgrade pip\n", + "!pip install unsloth vllm\n", + "!pip install --upgrade --no-deps \"trl>=0.14\" peft accelerate bitsandbytes\n", + "!pip install pydantic openenv huggingface_hub matplotlib" + ] + }, + { + "cell_type": "markdown", + "id": "8dd0d8092fe74a7c96281538738b07e2", + "metadata": {}, + "source": [ + "## 2. Authenticate with Hugging Face" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "72eea5119410473aa328ad9291626812", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "\n", + "try:\n", + " from google.colab import userdata\n", + "\n", + " HF_TOKEN = userdata.get(\"HF_TOKEN\")\n", + " os.environ[\"HF_TOKEN\"] = HF_TOKEN\n", + "except Exception:\n", + " from huggingface_hub import login\n", + "\n", + " login()\n", + " HF_TOKEN = os.environ.get(\"HF_TOKEN\", \"\")\n", + "\n", + "assert HF_TOKEN, \"HF_TOKEN required\"" + ] + }, + { + "cell_type": "markdown", + "id": "8edb47106e1a46a883d545849b8ab81b", + "metadata": {}, + "source": [ + "## 3. Clone CrisisWorldCortex (post-Cortex-Session-13 deploy)\n", + "\n", + "**TODO(conv-point):** the target Space repo at convergence point will have `cortex/*` populated through Session 13 (subagents + lenses + brains + council + metacognition + routing_policy + B3). Until then, the Cortex-dependent imports below will fail." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "10185d26023b46108eb7d9f57d49d2b3", + "metadata": {}, + "outputs": [], + "source": [ + "%%capture\n", + "!rm -rf /content/CrisisWorldCortex\n", + "!git clone https://huggingface.co/spaces/Angshuman28/CrisisWorldCortex /content/CrisisWorldCortex\n", + "%cd /content/CrisisWorldCortex\n", + "!pip install -e ." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8763a12b2bbd4a93a75aff182afb95dc", + "metadata": {}, + "outputs": [], + "source": [ + "import sys\n", + "\n", + "sys.path.insert(0, \"/content/CrisisWorldCortex\")\n", + "\n", + "\n", + "# TODO(conv-point): uncomment when Cortex Session 13 lands.\n", + "# from cortex.schemas import MetacognitionState, RoutingAction, RouterStep\n", + "# from cortex.routing_policy import RoutingPolicy\n", + "# from cortex.council import Council\n", + "# from baselines.cortex_fixed_router import B3CortexFixedRouter\n", + "print(\"Phase-2 training utilities OK; Cortex imports gated until Session 13\")" + ] + }, + { + "cell_type": "markdown", + "id": "7623eae2785240b9bd12b16a66d81610", + "metadata": {}, + "source": [ + "## 4. Load Qwen3-1.5B-Instruct (router) with LoRA\n", + "\n", + "Small enough to fit alongside frozen 7B/8B brain LLMs on a100-large (80GB). LoRA rank 16 — tighter than the B1 notebook's 32 because the action space is structured JSON (small effective vocabulary)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7cdc8c89c7104fffa095e18ddfef8986", + "metadata": {}, + "outputs": [], + "source": [ + "from unsloth import FastLanguageModel\n", + "\n", + "ROUTER_MODEL = \"unsloth/Qwen3-1.5B-Instruct-bnb-4bit\"\n", + "MAX_SEQ_LEN = 2048\n", + "\n", + "router_model, router_tokenizer = FastLanguageModel.from_pretrained(\n", + " model_name=ROUTER_MODEL,\n", + " max_seq_length=MAX_SEQ_LEN,\n", + " load_in_4bit=True,\n", + " fast_inference=True,\n", + " max_lora_rank=16,\n", + " gpu_memory_utilization=0.5, # share GPU with frozen brain LLMs at conv-point\n", + ")\n", + "\n", + "router_model = FastLanguageModel.get_peft_model(\n", + " router_model,\n", + " r=16,\n", + " target_modules=[\"q_proj\", \"k_proj\", \"v_proj\", \"o_proj\", \"gate_proj\", \"up_proj\", \"down_proj\"],\n", + " lora_alpha=32,\n", + " use_gradient_checkpointing=\"unsloth\",\n", + " random_state=42,\n", + ")\n", + "print(\"Router (Qwen3-1.5B + LoRA r=16) loaded\")" + ] + }, + { + "cell_type": "markdown", + "id": "b118ea5561624da68c537baed56e602f", + "metadata": {}, + "source": [ + "## 5. Featurization: MetacognitionState → NL prompt + RoutingAction schema\n", + "\n", + "Replaces the MLP version's `(24,) np.float32` featurization. The router consumes a natural-language summary; output is structured JSON validated against `cortex.schemas.RoutingAction`.\n", + "\n", + "**TODO(conv-point):** the schema below references `cortex.schemas.MetacognitionState` (Session 13).\n", + "Per Phase A `cortex/schemas.py` and `docs/CORTEX_ARCHITECTURE.md` §6, the NL summary covers the 11 documented fields + the phase string." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "938c804e27f84196a10c8828c723f798", + "metadata": {}, + "outputs": [], + "source": [ + "ROUTER_SYSTEM_PROMPT = \"\"\"You are the Cortex router for CrisisWorldCortex. You receive a metacognition\n", + "state summary describing the current deliberation phase and emit ONE routing action as JSON.\n", + "\n", + "Allowed action kinds (per cortex/CLAUDE.md):\n", + "- call_subagent: invoke a brain's subagent. Required: brain (epidemiology|logistics|governance),\n", + " subagent (world_modeler|planner|critic).\n", + "- request_challenge: cross-brain critique. Required: challenger_brain, target_brain.\n", + "- switch_phase: advance the phase machine. Required: new_phase (divergence|challenge|narrowing|convergence).\n", + "- preserve_dissent: tag a minority recommendation. Required: tag (string, max 80 chars).\n", + "- emit_outer_action: close the tick with a final action.\n", + "- stop_and_no_op: close the tick with a no-op.\n", + "\n", + "Hard caps (binding): ≤2 deliberation rounds/tick, ≤1 cross-brain challenge/tick,\n", + "≤1 critic call per brain/tick, ≤6000 token budget per tick.\n", + "\n", + "Output exactly one JSON object — no markdown fences, no prose around it.\"\"\"\n", + "\n", + "PHASE_NAMES = (\"divergence\", \"challenge\", \"narrowing\", \"convergence\")\n", + "\n", + "\n", + "def metacog_state_to_prompt(state) -> str:\n", + " \"\"\"Convert MetacognitionState → NL summary for the router.\n", + "\n", + " TODO(conv-point): change ``state`` typing to MetacognitionState\n", + " (cortex.schemas) once Session 13 lands. Duck-typed for now.\n", + " \"\"\"\n", + " return (\n", + " f\"Tick {getattr(state, 'tick', 0)}, round {getattr(state, 'round', 1)}, \"\n", + " f\"phase={getattr(state, 'phase', 'divergence')}.\\n\"\n", + " f\"Inter-brain agreement: {getattr(state, 'inter_brain_agreement', 0.0):.2f}.\\n\"\n", + " f\"Average confidence: {getattr(state, 'average_confidence', 0.0):.2f}.\\n\"\n", + " f\"Average evidence support: {getattr(state, 'average_evidence_support', 0.0):.2f}.\\n\"\n", + " f\"Novelty yield (last round): {getattr(state, 'novelty_yield_last_round', 0.0):.2f}.\\n\"\n", + " f\"Collapse suspicion: {getattr(state, 'collapse_suspicion', 0.0):.2f}.\\n\"\n", + " f\"Budget remaining: {getattr(state, 'budget_remaining_frac', 1.0):.0%}.\\n\"\n", + " f\"Urgency: {getattr(state, 'urgency', 0.0):.2f}.\\n\"\n", + " f\"Preserved dissent count: {getattr(state, 'preserved_dissent_count', 0)}.\\n\"\n", + " f\"Cross-brain challenge used this tick: \"\n", + " f\"{bool(getattr(state, 'challenge_used_this_tick', 0))}.\\n\\n\"\n", + " f\"Choose the next routing action.\"\n", + " )\n", + "\n", + "\n", + "def make_router_chat_prompt(state) -> str:\n", + " return router_tokenizer.apply_chat_template(\n", + " [\n", + " {\"role\": \"system\", \"content\": ROUTER_SYSTEM_PROMPT},\n", + " {\"role\": \"user\", \"content\": metacog_state_to_prompt(state)},\n", + " ],\n", + " tokenize=False,\n", + " add_generation_prompt=True,\n", + " )" + ] + }, + { + "cell_type": "markdown", + "id": "504fb2a444614c0babb325280ed9130a", + "metadata": {}, + "source": [ + "## 6. Connect to the deployed CrisisWorld env\n", + "\n", + "Same env client pattern as the B1 notebook." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "59bbdb311c014d738909a11f9e486628", + "metadata": {}, + "outputs": [], + "source": [ + "from CrisisWorldCortex.client import CrisisworldcortexEnv\n", + "\n", + "ENV_URL = \"https://angshuman28-crisisworldcortex.hf.space\"\n", + "TASKS = (\"outbreak_easy\", \"outbreak_medium\", \"outbreak_hard\")\n", + "EPISODE_TICKS = 12\n", + "\n", + "\n", + "def make_env() -> CrisisworldcortexEnv:\n", + " return CrisisworldcortexEnv(base_url=ENV_URL)\n", + "\n", + "\n", + "_test_env = make_env()\n", + "_obs = _test_env.reset(task_name=\"outbreak_easy\", seed=0, max_ticks=EPISODE_TICKS)\n", + "print(f\"Env OK. Initial tick={_obs.tick}, regions={[r.region for r in _obs.regions]}\")" + ] + }, + { + "cell_type": "markdown", + "id": "b43b363d81ae4b689946ece5c682cd59", + "metadata": {}, + "source": [ + "## 7. Build training-data prompt set from B3 deterministic-router trajectories\n", + "\n", + "**TODO(conv-point):** B3CortexFixedRouter runs ~50 episodes. Each `RouterStep`'s `MetacognitionState` becomes a router-prompt; the GRPO completion is the router's emitted JSON.\n", + "\n", + "Replaces the MLP version's action-vocab + B3-trajectory-to-tuple pipeline. Same RolloutBuffer, different content shape." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8a65eabff63a45729fe45fb5ade58bdc", + "metadata": {}, + "outputs": [], + "source": [ + "from datasets import Dataset\n", + "\n", + "TRAIN_EPISODES = 50\n", + "\n", + "\n", + "def collect_b3_router_prompts(num_episodes: int = TRAIN_EPISODES) -> Dataset:\n", + " \"\"\"Run B3 and convert each RouterStep into a (prompt, task, seed) row.\n", + "\n", + " TODO(conv-point): uncomment the body when B3CortexFixedRouter ships.\n", + " \"\"\"\n", + " rows = {\"prompt\": [], \"task\": [], \"seed\": []}\n", + " # TODO(conv-point):\n", + " # b3 = B3CortexFixedRouter(env=make_env())\n", + " # for ep in range(num_episodes):\n", + " # trajectory = b3.run_episode(task=\"outbreak_easy\", seed=ep)\n", + " # for router_step in trajectory.router_steps:\n", + " # rows[\"prompt\"].append(make_router_chat_prompt(router_step.metacognition_state))\n", + " # rows[\"task\"].append(\"outbreak_easy\")\n", + " # rows[\"seed\"].append(ep)\n", + " return (\n", + " Dataset.from_dict(rows)\n", + " if rows[\"prompt\"]\n", + " else Dataset.from_dict(\n", + " {\"prompt\": [\"placeholder until conv-point\"], \"task\": [\"outbreak_easy\"], \"seed\": [0]}\n", + " )\n", + " )\n", + "\n", + "\n", + "train_dataset = collect_b3_router_prompts()\n", + "print(f\"Dataset: {len(train_dataset)} rows (will be ~{TRAIN_EPISODES * 8} at conv-point)\")" + ] + }, + { + "cell_type": "markdown", + "id": "c3933fab20d04ec698c2621248eb3be0", + "metadata": {}, + "source": [ + "## 8. Reward function: full-episode rollout per (prompt, completion)\n", + "\n", + "Uses Phase-1 `outer_reward` summed across the episode, with Phase-2 `shape_reward` token-budget penalty.\n", + "\n", + "**TODO(conv-point):** the rollout loop calls `Council.step` with the trainable router policy. Until Session 13 lands, the loop is stubbed and returns 0.0 for every (prompt, completion)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4dd4641cc4064e0191573fe9c69df29b", + "metadata": {}, + "outputs": [], + "source": [ + "def cortex_router_reward(prompts, completions, task, seed, **_kwargs):\n", + " \"\"\"Reward = sum(per-tick obs.reward) over a full episode driven by the\n", + " trainable router's emitted RoutingAction JSON.\n", + "\n", + " Each (prompt, completion) pair represents ONE router decision; the\n", + " full episode reward is shared across all router decisions in that\n", + " episode (GRPO group-relative advantage handles credit assignment).\n", + "\n", + " TODO(conv-point): replace the stub body with the real Council-driven\n", + " rollout once Session 13 ships.\n", + " \"\"\"\n", + " rewards = []\n", + " for completion, t, s in zip(completions, task, seed):\n", + " # TODO(conv-point):\n", + " # try:\n", + " # routing_action = RoutingAction.model_validate_json(completion)\n", + " # except ValidationError:\n", + " # rewards.append(-1.0) # invalid JSON → terminal-equivalent penalty\n", + " # continue\n", + " # council = Council(routing_policy=trainable_router_from(routing_action),\n", + " # env=make_env(), brains=cortex_brains)\n", + " # episode_return = council.run_episode(task=t, seed=int(s)).total_reward\n", + " # rewards.append(episode_return)\n", + " rewards.append(0.0) # stub\n", + " return rewards" + ] + }, + { + "cell_type": "markdown", + "id": "8309879909854d7188b41380fd92a7c3", + "metadata": {}, + "source": [ + "## 9. GRPO training\n", + "\n", + "Same TRL `GRPOTrainer` shape as the B1 notebook. ~300 steps × group size 4 = ~1200 router decisions × full-episode rollouts. Wall-clock ~1.5 hours on a100-large at convergence point (dominated by brain-LLM rollouts, not router fine-tuning)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3ed186c9a28b402fb0bc4494df01f08d", + "metadata": {}, + "outputs": [], + "source": [ + "from trl import GRPOConfig, GRPOTrainer\n", + "\n", + "MAX_TRAIN_STEPS = 300\n", + "GROUP_SIZE = 4\n", + "MAX_PROMPT_LEN = 512\n", + "MAX_COMPLETION_LEN = 256 # router output is structured JSON (M-FR-11)\n", + "\n", + "training_args = GRPOConfig(\n", + " output_dir=\"/content/cortex_router_grpo_output\",\n", + " learning_rate=5e-6,\n", + " per_device_train_batch_size=GROUP_SIZE,\n", + " gradient_accumulation_steps=1,\n", + " num_generations=GROUP_SIZE,\n", + " max_prompt_length=MAX_PROMPT_LEN,\n", + " max_completion_length=MAX_COMPLETION_LEN,\n", + " max_steps=MAX_TRAIN_STEPS,\n", + " save_steps=100,\n", + " logging_steps=5,\n", + " report_to=\"none\",\n", + " bf16=True,\n", + " optim=\"adamw_8bit\",\n", + " temperature=0.8,\n", + " use_vllm=True,\n", + " vllm_mode=\"colocate\",\n", + " seed=42,\n", + ")\n", + "\n", + "trainer = GRPOTrainer(\n", + " model=router_model,\n", + " processing_class=router_tokenizer,\n", + " reward_funcs=[cortex_router_reward],\n", + " args=training_args,\n", + " train_dataset=train_dataset,\n", + ")\n", + "print(\"Router GRPOTrainer constructed.\")\n", + "print(\"# TODO(conv-point): uncomment trainer.train() once Cortex Session 13 lands.\")\n", + "# trainer.train()" + ] + }, + { + "cell_type": "markdown", + "id": "cb1e1581032b452c9409d6c6813c49d1", + "metadata": {}, + "source": [ + "## 10. Save the trained router LoRA to HF Hub" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "379cbbc1e968416e875cc15c1202d7eb", + "metadata": {}, + "outputs": [], + "source": [ + "from huggingface_hub import HfApi\n", + "\n", + "HUB_REPO = \"Angshuman28/crisisworld-cortex-router-llm\"\n", + "\n", + "router_model.save_pretrained(\"/content/cortex_router_lora\")\n", + "router_tokenizer.save_pretrained(\"/content/cortex_router_lora\")\n", + "\n", + "api = HfApi()\n", + "api.create_repo(HUB_REPO, exist_ok=True, repo_type=\"model\", private=False, token=HF_TOKEN)\n", + "api.upload_folder(\n", + " folder_path=\"/content/cortex_router_lora\",\n", + " repo_id=HUB_REPO,\n", + " repo_type=\"model\",\n", + " token=HF_TOKEN,\n", + ")\n", + "print(f\"Saved to https://huggingface.co/{HUB_REPO}\")" + ] + }, + { + "cell_type": "markdown", + "id": "277c27b1587741f2af2001be3712ef0d", + "metadata": {}, + "source": [ + "## 11. Eval: B6 (trained LLM router) vs B3 (deterministic) on 3 tasks\n", + "\n", + "**TODO(conv-point):** the comparison loop below requires both B3CortexFixedRouter (deterministic) and a way to swap the trainable router into Council.routing_policy. The eval is the headline result for the convergence point — its sign decides whether B6 ships or B3 ships per the Phase 7 hard-exit gate.\n", + "\n", + "Decision rule (per spec): if reward over training steps is INCREASING, ship B6. Else ship B3." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "db7b79bc585a40fcaf58bf750017e135", + "metadata": {}, + "outputs": [], + "source": [ + "# TODO(conv-point):\n", + "# from cortex.routing_policy import DeterministicRouter, TrainableRouter\n", + "# from cortex.council import Council\n", + "# from baselines.cortex_fixed_router import B3CortexFixedRouter\n", + "#\n", + "# def run_b6_episode(task, seed):\n", + "# trainable = TrainableRouter(\n", + "# model=router_model,\n", + "# tokenizer=router_tokenizer,\n", + "# system_prompt=ROUTER_SYSTEM_PROMPT,\n", + "# featurize=metacog_state_to_prompt,\n", + "# )\n", + "# council = Council(routing_policy=trainable, env=make_env(), brains=cortex_brains)\n", + "# return council.run_episode(task=task, seed=seed).total_reward\n", + "#\n", + "# def run_b3_episode(task, seed):\n", + "# b3 = B3CortexFixedRouter(env=make_env())\n", + "# return b3.run_episode(task=task, seed=seed).total_reward\n", + "#\n", + "# b6_results = {t: run_b6_episode(t, 0) for t in TASKS}\n", + "# b3_results = {t: run_b3_episode(t, 0) for t in TASKS}\n", + "# print(f\"B6 (trained LLM router): {b6_results}\")\n", + "# print(f\"B3 (deterministic): {b3_results}\")\n", + "\n", + "print(\"Eval cell skeleton — uncomment when Cortex Session 13 ships\")" + ] + }, + { + "cell_type": "markdown", + "id": "916684f9a58a4a2aa5f864670399430d", + "metadata": {}, + "source": [ + "## 12. Plot training reward curve\n", + "\n", + "Phase 7 hard-exit gate watches this curve. If reward is increasing across training steps → ship B6. Else → ship B3." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1671c31a24314836a5b85d7ef7fbf015", + "metadata": {}, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "\n", + "log_path = \"/content/cortex_router_grpo_output/trainer_state.json\"\n", + "if os.path.exists(log_path):\n", + " import json as _json\n", + "\n", + " with open(log_path) as fh:\n", + " state = _json.load(fh)\n", + " history = state.get(\"log_history\", [])\n", + " rewards = [entry[\"reward\"] for entry in history if \"reward\" in entry]\n", + " if rewards:\n", + " fig, ax = plt.subplots(figsize=(8, 4))\n", + " ax.plot(rewards)\n", + " ax.set_xlabel(\"GRPO step\")\n", + " ax.set_ylabel(\"Mean reward\")\n", + " ax.set_title(\"Cortex small-LLM router — training reward\")\n", + " plt.tight_layout()\n", + " plt.show()\n", + " else:\n", + " print(\"No reward entries yet — uncomment trainer.train() at conv-point\")\n", + "else:\n", + " print(\"No trainer state yet — uncomment trainer.train() at conv-point\")" + ] + } + ], + "metadata": { + "accelerator": "GPU", + "colab": { + "gpuType": "A100", + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.10" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/openenv.yaml b/openenv.yaml index 76c24e3eb35c72fe7beb19dcf3c1646517707b6f..a5af7b1cdae13172bdd8808d2c1ab2e8f12b9846 100644 --- a/openenv.yaml +++ b/openenv.yaml @@ -1,7 +1,7 @@ -spec_version: 1 -name: CrisisWorldCortex -type: space -runtime: fastapi -app: server.app:app -port: 8000 - +spec_version: 1 +name: CrisisWorldCortex +type: space +runtime: fastapi +app: server.app:app +port: 8000 + diff --git a/openenv_CrisisWorldCortex.egg-info/PKG-INFO b/openenv_CrisisWorldCortex.egg-info/PKG-INFO new file mode 100644 index 0000000000000000000000000000000000000000..8e2c244732135369df9f5cc616c170377b703d96 --- /dev/null +++ b/openenv_CrisisWorldCortex.egg-info/PKG-INFO @@ -0,0 +1,13 @@ +Metadata-Version: 2.4 +Name: openenv-CrisisWorldCortex +Version: 0.1.0 +Summary: Crisisworldcortex environment for OpenEnv +Requires-Python: >=3.10 +Requires-Dist: openenv-core[core]==0.2.3 +Requires-Dist: openai<3.0,>=2.0 +Requires-Dist: python-dotenv>=1.0.0 +Provides-Extra: dev +Requires-Dist: pre-commit>=4.0.0; extra == "dev" +Requires-Dist: pytest>=8.0.0; extra == "dev" +Requires-Dist: pytest-cov>=4.0.0; extra == "dev" +Requires-Dist: ruff>=0.8.0; extra == "dev" diff --git a/openenv_CrisisWorldCortex.egg-info/SOURCES.txt b/openenv_CrisisWorldCortex.egg-info/SOURCES.txt new file mode 100644 index 0000000000000000000000000000000000000000..57df02b4ebf1bf2bd33624bdaa676b629ec5027a --- /dev/null +++ b/openenv_CrisisWorldCortex.egg-info/SOURCES.txt @@ -0,0 +1,48 @@ +README.md +__init__.py +client.py +inference.py +models.py +pyproject.toml +./__init__.py +./client.py +./inference.py +./models.py +openenv_CrisisWorldCortex.egg-info/PKG-INFO +openenv_CrisisWorldCortex.egg-info/SOURCES.txt +openenv_CrisisWorldCortex.egg-info/dependency_links.txt +openenv_CrisisWorldCortex.egg-info/entry_points.txt +openenv_CrisisWorldCortex.egg-info/requires.txt +openenv_CrisisWorldCortex.egg-info/top_level.txt +server/CrisisWorldCortex_environment.py +server/__init__.py +server/app.py +tests/test_actions_round_trip.py +tests/test_baseline_b1.py +tests/test_baseline_b2.py +tests/test_cortex_brain_executive.py +tests/test_cortex_brain_smoke.py +tests/test_cortex_lenses.py +tests/test_cortex_perception.py +tests/test_cortex_subagents.py +tests/test_env_reset_kwargs.py +tests/test_env_step_reward_wiring.py +tests/test_import_graph.py +tests/test_legal_constraint_enforcement.py +tests/test_llm_client.py +tests/test_observation_no_latent_leak.py +tests/test_outer_reward_in_range.py +tests/test_outer_reward_non_constancy.py +tests/test_outer_reward_terminal_bonus.py +tests/test_package_exports.py +tests/test_reward_signal_quality.py +tests/test_schemas_roundtrip.py +tests/test_simulator_determinism.py +tests/test_simulator_random_episode.py +tests/test_simulator_task_configs.py +tests/test_smoke_env.py +tests/test_stdout_format.py +tests/test_synthetic_rejection_payload.py +tests/test_training_eval_metrics.py +tests/test_training_reward_shaping.py +tests/test_training_rollout_buffer.py \ No newline at end of file diff --git a/openenv_CrisisWorldCortex.egg-info/dependency_links.txt b/openenv_CrisisWorldCortex.egg-info/dependency_links.txt new file mode 100644 index 0000000000000000000000000000000000000000..8b137891791fe96927ad78e64b0aad7bded08bdc --- /dev/null +++ b/openenv_CrisisWorldCortex.egg-info/dependency_links.txt @@ -0,0 +1 @@ + diff --git a/openenv_CrisisWorldCortex.egg-info/entry_points.txt b/openenv_CrisisWorldCortex.egg-info/entry_points.txt new file mode 100644 index 0000000000000000000000000000000000000000..81af91268b4bee194cb07a5df3b995f50167034e --- /dev/null +++ b/openenv_CrisisWorldCortex.egg-info/entry_points.txt @@ -0,0 +1,2 @@ +[console_scripts] +server = CrisisWorldCortex.server.app:main diff --git a/openenv_CrisisWorldCortex.egg-info/requires.txt b/openenv_CrisisWorldCortex.egg-info/requires.txt new file mode 100644 index 0000000000000000000000000000000000000000..8a466289665d9c803e569a8854d16f29984e624d --- /dev/null +++ b/openenv_CrisisWorldCortex.egg-info/requires.txt @@ -0,0 +1,9 @@ +openenv-core[core]==0.2.3 +openai<3.0,>=2.0 +python-dotenv>=1.0.0 + +[dev] +pre-commit>=4.0.0 +pytest>=8.0.0 +pytest-cov>=4.0.0 +ruff>=0.8.0 diff --git a/openenv_CrisisWorldCortex.egg-info/top_level.txt b/openenv_CrisisWorldCortex.egg-info/top_level.txt new file mode 100644 index 0000000000000000000000000000000000000000..384b20ea02864efd9a4a124d3c50339706e873e1 --- /dev/null +++ b/openenv_CrisisWorldCortex.egg-info/top_level.txt @@ -0,0 +1 @@ +CrisisWorldCortex diff --git a/pyproject.toml b/pyproject.toml index df4b221ddf6bbf1f3c5072e3533a847da58bfab8..50b63073e10f52a3254a9718c16d5ac3be0fc586 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,6 +25,7 @@ dependencies = [ # future uv sync from silently pulling 3.x. Bump explicitly when 3.0 # ships and we've verified compatibility. "openai>=2.0,<3.0", + "python-dotenv>=1.0.0" ] [project.optional-dependencies] diff --git a/server/CLAUDE.md b/server/CLAUDE.md index 9dd0192dc3f2014cb7b1b3344dca1ef24ac72da5..42259c6136f6c82337c681b9fbdad873fd881355 100644 --- a/server/CLAUDE.md +++ b/server/CLAUDE.md @@ -15,33 +15,24 @@ Cortex code (brains, subagents, router, metacognition), LLM clients, training lo ## Import rule (binding) -**Inside `server/`, use package-relative imports for server internals**: -`from .simulator import ...` from one-level modules and -`from ..simulator import ...` from nested modules. This works when the -server is loaded as either `CrisisWorldCortex.server.*` (`uv run server`) -or top-level `server.*` (Docker / `uvicorn server.app:app`). Never use -`from CrisisWorldCortex.server...` for server-internal imports. +**Inside `server/`, use `from server.simulator import …`, never `from CrisisWorldCortex.server.simulator import …`.** Same rule for `server.graders`. -For `models`, use the canonical package path in every server module that needs it: +For `models`, use the dual-import fallback in every new server module that needs it: ```python -from CrisisWorldCortex.models import CrisisworldcortexAction, CrisisworldcortexObservation +try: + from ..models import CrisisworldcortexAction, CrisisworldcortexObservation +except (ImportError, ModuleNotFoundError): + from models import CrisisworldcortexAction, CrisisworldcortexObservation ``` -The server runs under multiple import contexts (`uv run server`, -`uvicorn server.app:app`, Docker `cd /app/env && uvicorn server.app:app`). -Canonical wire imports keep Pydantic model identity stable across those modes. +The server runs under ≥ 3 import contexts (`python -m server.app`, `uvicorn server.app:app`, Docker `cd /app/env && uvicorn server.app:app`); the fallback is load-bearing. -**Wire-type imports from deep modules (binding)**: two-or-more-levels-deep -files (`server/simulator/*`, `server/graders/*`) must still use -`from CrisisWorldCortex.models import …`; `..models` from those depths -resolves to a non-existent `CrisisWorldCortex.server.models`, and a bare -fallback loads a second `models` module. +**Wire-type imports from deep modules (binding)**: the dual-import fallback above only works for files **one level** inside `server/` (`server/CrisisWorldCortex_environment.py`, `server/app.py`). Two-or-more-levels-deep files (`server/simulator/*`, `server/graders/*`) **must** use `from CrisisWorldCortex.models import …` directly — `..models` from those depths resolves to a non-existent `CrisisWorldCortex.server.models`, the fallback fires, and bare `models` loads as a separate `sys.modules` entry, breaking Pydantic discriminator validation against types imported via the canonical path. Session 5a's `server/simulator/seir_model.py` and `server/simulator/tasks.py` document this with inline comments. ## Allowed imports -`CrisisWorldCortex.models`, package-relative `server/simulator/*` and -`server/graders/*`, `openenv.core.*`, stdlib, FastAPI, Pydantic, numpy. +`models` (via dual-import fallback), `openenv.core.*`, `server/simulator/*`, `server/graders/*`, stdlib, FastAPI, Pydantic, numpy. ## Forbidden imports diff --git a/server/CrisisWorldCortex_environment.py b/server/CrisisWorldCortex_environment.py index c368ac7fe104a173caf012f0ef9040d0114c16ba..92caa8d858e9519ed76300c542e8691d0e06eb30 100644 --- a/server/CrisisWorldCortex_environment.py +++ b/server/CrisisWorldCortex_environment.py @@ -121,10 +121,26 @@ class CrisisworldcortexEnvironment(Environment): ) self._state.step_count += 1 self._world_state = apply_tick(self._world_state, action.action) + # Parse-failure terminal contract (design §19, Phase-1 restoration): + # the synthetic parse_failure_marker (PublicCommunication with + # honesty=0.0, magic-string discriminator per Phase-A M3-B) ends + # the episode as state.terminal = "failure". apply_tick may have + # set terminal to "none"/"success"/"timeout" via the SEIR rules; + # we override here because parse-failure is a harness-level fault, + # not a simulator-level event. + payload = action.action + if ( + payload.kind == "public_communication" + and getattr(payload, "honesty", None) == 0.0 + and self._world_state.recent_action_log + and not self._world_state.recent_action_log[-1].accepted + ): + self._world_state.terminal = "failure" obs = make_observation(self._world_state) - # Per design §15: r_outer is the only env-side reward signal, in [0,1]. - # Terminal bonus (+/-0.20) is composed downstream by the trainer per - # design §14.3 — never bundled into obs.reward. + # Per design §15: r_outer is the only env-side reward signal, in + # [-1.0, 1.0] post-Phase-1 (was [0, 1]). Terminal bonus (+/-0.20) + # is composed downstream by the trainer per design §14.3 — never + # bundled into obs.reward. obs.reward = outer_reward(self._world_state, action.action) return obs diff --git a/server/__init__.py b/server/__init__.py index 2c074cae1197c3922c4df82b0fdf78f515671d11..648287bf7213bce8ff36fdcf08c7d7ebc7da8c39 100644 --- a/server/__init__.py +++ b/server/__init__.py @@ -1,11 +1,11 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# All rights reserved. -# -# This source code is licensed under the BSD-style license found in the -# LICENSE file in the root directory of this source tree. - -"""Crisisworldcortex environment server components.""" - -from .CrisisWorldCortex_environment import CrisisworldcortexEnvironment - -__all__ = ["CrisisworldcortexEnvironment"] +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +"""Crisisworldcortex environment server components.""" + +from .CrisisWorldCortex_environment import CrisisworldcortexEnvironment + +__all__ = ["CrisisworldcortexEnvironment"] diff --git a/server/app.py b/server/app.py index f3ed0bbead909b9a4c9e6086d286a22781ee6adc..6a0b202cd95398d2c22881329b23d061ca9e5147 100644 --- a/server/app.py +++ b/server/app.py @@ -29,8 +29,8 @@ Usage: """ try: - # from openenv.core.env_server.http_server import create_app - from openenv.core.env_server import create_web_interface_app as create_app + from openenv.core.env_server.http_server import create_app + # from openenv.core.env_server import create_web_interface_app as create_app except Exception as e: # pragma: no cover raise ImportError( "openenv is required for the web interface. Install dependencies with '\n uv sync\n'" diff --git a/server/graders/outer_reward.py b/server/graders/outer_reward.py index 5e10875fb5f4bf6577e988da5841e24916f42ef1..42deb52ce887c4ce31a6f2eee0c1e4634ce23468 100644 --- a/server/graders/outer_reward.py +++ b/server/graders/outer_reward.py @@ -5,16 +5,29 @@ # LICENSE file in the root directory of this source tree. """ -Outer reward grader for CrisisWorld (design §15). +Outer reward grader for CrisisWorld (design §15 + §19, post-Phase-1 fix). Public API (re-exported via ``server/graders/__init__.py``): - ``outer_reward(state, action) -> float`` — 6-component weighted score in - ``[0.0, 1.0]``, computed on post-``apply_tick`` state. The only env-side - reward signal per ``server/CLAUDE.md`` (binding contract). + ``[-1.0, 1.0]`` (Workstream-B Phase-1 range relaxation; was + ``[0.0, 1.0]``). Negative values come from ``r_policy = -0.5`` on rejected + actions and ``r_policy = -1.0`` on parse-failure markers (per design §19). + Computed on post-``apply_tick`` state. Still the only env-side reward + signal per ``server/CLAUDE.md``. - ``terminal_bonus(state) -> float`` — episode-end ±0.20 / 0.0 bonus, composed by trainer in ``training/reward_shaping.py`` per design §14.3 - (``episode_return = Σ_t r_outer + terminal_bonus``). Kept separate so the - per-tick ``r_outer`` stays inside ``[0.0, 1.0]``. + (``episode_return = Σ_t r_outer + terminal_bonus``). Kept separate from + the per-tick scalar. + +Phase-1 changes (Workstream B): + - Steeper sensitivity on ``r_infect`` (``× 20``) and ``r_hosp`` (``× 10``) + so the gentle outbreak_easy task still produces gradient. + - Continuous ``r_casc`` (``1 - max(I)/0.30`` clamped) replaces binary. + - ``r_policy`` ∈ {-1.0 (parse-failure), -0.5 (rejected), 0.0 (accepted + no_op), +1.0 (accepted real action)} restoring §19 magnitudes. + - Weight redistribution: W_POLICY 0.12 → 0.35 (signal-driver); W_TIME + 0.18 → 0.05 (action-independent noise); other components rebalanced. + - Final ``[0,1]`` clamp dropped (Phase-A M2-A). Wire-protocol imports use the absolute path ``CrisisWorldCortex.models`` because this file lives two levels deep inside ``server/`` — see @@ -38,25 +51,42 @@ from ..simulator import ( ) # ============================================================================ -# Component weights (design §15; sum to 1.00) +# Component weights (design §15 + Phase-1 redistribution; sum to 1.00) # ============================================================================ -W_INFECT = 0.35 -W_TIME = 0.18 -W_HOSP = 0.17 -W_CASC = 0.15 -W_POLICY = 0.12 -W_FAIR = 0.03 +W_INFECT = 0.15 # was 0.35; iter-1 reduced because outbreak_easy keeps mean(I) tiny +W_TIME = 0.05 # was 0.18; action-independent, no signal value +W_HOSP = 0.10 # was 0.17; iter-1 reduction (gentle env keeps hosp_load low) +W_CASC = 0.10 # was 0.15; iter-1 reduction +W_POLICY = 0.55 # was 0.12; iter-1 dominant signal driver — accepted-real vs no_op vs rejected +W_FAIR = 0.05 # was 0.03; tiny boost + +# Steepness coefficients (Phase-A M6, ONE-iteration tentative). +# r_infect ≈ 0 when mean(I) >= 0.05; near 1 when mean(I) <= 0.0 → strong gradient. +# r_hosp similarly sensitive to mean hospital_load. +R_INFECT_STEEPNESS = 20.0 +R_HOSP_STEEPNESS = 10.0 + +# r_policy values per design §19 (Phase-1 restoration). +R_POLICY_PARSE_FAILURE = -1.0 # synthetic parse-failure marker +R_POLICY_REJECTED = -0.5 # well-formed-illegal (V2 / legal-violation) +R_POLICY_NOOP_ACCEPTED = 0.0 # accepted no-op (legal but inactive) +R_POLICY_REAL_ACCEPTED = 1.0 # accepted real intervention # Terminal-bonus magnitudes (design §14.3 / §15). TERMINAL_BONUS_SUCCESS = 0.20 TERMINAL_BONUS_FAILURE = -0.20 -# r_casc threshold: any region above this contributes a "hot" count; -# matches the catastrophic-region threshold from design §6.4 so the -# cascade-control signal aligns with the failure terminal. +# r_casc threshold: at max(I) >= this, r_casc = 0 (catastrophe imminent). +# Matches design §6.4's catastrophic-infection threshold so cascade +# signal aligns with the failure terminal. R_CASC_HOT_THRESHOLD = CATASTROPHIC_INFECTION_THRESHOLD +# Magic-string discriminator for parse-failure marker (Phase-A M3-B): +# baselines.flat_agent.parse_failure_marker emits PublicCommunication with +# honesty=0.0; intentional V2 attempts use honesty > 0.0. +PARSE_FAILURE_HONESTY_SENTINEL = 0.0 + def _clamp01(x: float) -> float: if x < 0.0: @@ -76,34 +106,56 @@ def _hospital_load(region_I: float) -> float: return _clamp01(region_I * HOSPITALIZATION_FRACTION_OF_I / HOSPITAL_CAPACITY_FRACTION) +def _r_policy_value(action: OuterActionPayload, accepted: bool) -> float: + """Compute ``r_policy`` per design §19 four-state contract. + + Returns one of {-1.0, -0.5, 0.0, +1.0} based on (action.kind, accepted). + Parse-failure detection uses the ``honesty == 0.0`` sentinel on a + rejected ``PublicCommunication`` payload (Phase-A M3-B magic string). + """ + if not accepted: + # Rejected branch. Distinguish parse-failure marker from intentional + # V2-PublicCommunication / legal-violation rejection. + if ( + action.kind == "public_communication" + and getattr(action, "honesty", None) == PARSE_FAILURE_HONESTY_SENTINEL + ): + return R_POLICY_PARSE_FAILURE + return R_POLICY_REJECTED + # Accepted branch. + if action.kind == "no_op": + return R_POLICY_NOOP_ACCEPTED + return R_POLICY_REAL_ACCEPTED + + def outer_reward(state: WorldState, action: OuterActionPayload) -> float: - """Compute per-tick outer reward in ``[0.0, 1.0]``. + """Compute per-tick outer reward in ``[-1.0, 1.0]`` (post-Phase-1 range). Read post-``apply_tick`` state: ``state.regions[*].I`` is the just-stepped ground-truth infection fraction. ``recent_action_log[-1]`` holds the just-dispatched action's acceptance flag. - Six components (design §15): - r_infect = 1 - mean(I) # weight 0.35 - r_time = 1 - tick / max_ticks # weight 0.18 - r_hosp = 1 - mean(hospital_load) # weight 0.17 - r_casc = 1 if no region exceeds 0.30 else 0 # weight 0.15 - r_policy = 1 if last action accepted else 0 # weight 0.12 - r_fair = 1 - pstdev(I) # weight 0.03 + Six components (design §15 + Phase-1 fix): + r_infect = max(0, 1 - 20 × mean(I)) # weight 0.25 + r_time = 1 - tick / max_ticks # weight 0.05 + r_hosp = max(0, 1 - 10 × mean(hospital_load)) # weight 0.15 + r_casc = max(0, 1 - max(I) / 0.30) # weight 0.15 + r_policy = {-1.0, -0.5, 0.0, +1.0} per §19 # weight 0.35 + r_fair = 1 - pstdev(I) # weight 0.05 The ``action`` argument is the action just dispatched. We read its acceptance flag from ``state.recent_action_log[-1]`` rather than re-dispatching — the simulator already recorded it, and re-dispatch - would mutate state. ``action`` is kept in the signature for design - parity with ``training_reward(trajectory)`` and to support future - action-shape-dependent shaping without a signature break. + would mutate state. """ if not state.regions: return 0.0 I_values = [r.I for r in state.regions] + mean_I = sum(I_values) / len(I_values) - r_infect = _clamp01(1.0 - sum(I_values) / len(I_values)) + # r_infect: steepened so gentle outbreak_easy still produces gradient. + r_infect = _clamp01(1.0 - R_INFECT_STEEPNESS * mean_I) if state.max_ticks > 0: r_time = _clamp01(1.0 - state.tick / state.max_ticks) @@ -111,16 +163,22 @@ def outer_reward(state: WorldState, action: OuterActionPayload) -> float: r_time = 0.0 hosp_loads = [_hospital_load(I) for I in I_values] - r_hosp = _clamp01(1.0 - sum(hosp_loads) / len(hosp_loads)) + mean_hosp = sum(hosp_loads) / len(hosp_loads) + r_hosp = _clamp01(1.0 - R_HOSP_STEEPNESS * mean_hosp) - hot_regions = sum(1 for I in I_values if I > R_CASC_HOT_THRESHOLD) - r_casc = 1.0 if hot_regions == 0 else 0.0 + # r_casc: continuous ramp (1.0 at max(I)=0; 0.0 at max(I) >= threshold). + max_I = max(I_values) + if R_CASC_HOT_THRESHOLD > 0: + r_casc = _clamp01(1.0 - max_I / R_CASC_HOT_THRESHOLD) + else: + r_casc = 0.0 + # r_policy: design §19 four-state contract. if state.recent_action_log: last_entry = state.recent_action_log[-1] - r_policy = 1.0 if last_entry.accepted else 0.0 + r_policy = _r_policy_value(last_entry.action, last_entry.accepted) else: - r_policy = 1.0 # No action dispatched yet → no penalty. + r_policy = R_POLICY_REAL_ACCEPTED # No action dispatched yet → no penalty. if len(I_values) >= 2: r_fair = _clamp01(1.0 - statistics.pstdev(I_values)) @@ -135,8 +193,12 @@ def outer_reward(state: WorldState, action: OuterActionPayload) -> float: + W_POLICY * r_policy + W_FAIR * r_fair ) - # Final clamp guards against floating-point drift past 1.0. - return _clamp01(score) + # Final clamp to [-1.0, 1.0] (no longer [0, 1] — M2-A drops the floor). + if score < -1.0: + return -1.0 + if score > 1.0: + return 1.0 + return score def terminal_bonus(state: WorldState) -> float: diff --git a/server/requirements.txt b/server/requirements.txt index 65b1c22b3db715ed9d63b9ad06cd4afb0d9412c5..a458250888125476be6407f1ffb26632235a9d15 100644 --- a/server/requirements.txt +++ b/server/requirements.txt @@ -1,6 +1,6 @@ -openenv[core]>=0.2.0 -fastapi>=0.115.0 -uvicorn>=0.24.0 - - - +openenv[core]>=0.2.0 +fastapi>=0.115.0 +uvicorn>=0.24.0 + + + diff --git a/tests/CLAUDE.md b/tests/CLAUDE.md index 86c31d3891d255c7e070203b1ef1c97d5e033db1..91539607d385364b0b7b8ba54b27b0aa21045275 100644 --- a/tests/CLAUDE.md +++ b/tests/CLAUDE.md @@ -1,48 +1,48 @@ -# tests/CLAUDE.md - -Test surface per subsystem. Smoke bar first, boundary tests next, coverage last. - -## Belongs here - -- `conftest.py` — repo root on `sys.path` for bare-name imports. -- One test module per subsystem boundary (table below). - -## Does not belong here - -Helpers that mutate real graders, simulator state, or disk. Fixtures that hit the live HF Space — mock or `pytest.skip`. - -## Run commands - -```bash -uv run python -m pytest tests/ -v # all -uv run python -m pytest tests/test_smoke_env.py::test_reset_returns_valid_observation -v # one -uv run python -m pytest --cov tests/ # coverage -``` - -## Required tests — each maps to exactly one subsystem contract - -| File | Scope | Asserts | -|---|---|---| -| `test_package_exports.py` | wire package | Root `__init__` re-exports `CrisisworldcortexAction/Observation/Env`. | -| `test_smoke_env.py` | `server/` env | `reset()` / `step()` return a valid `CrisisworldcortexObservation`. | -| `test_actions_round_trip.py` | `server/` env | 6 MVP outer actions round-trip; `public_communication` is rejected at runtime. | -| `test_reward_shape.py` | `server/graders/` | Every grader returns values in `[0.0, 1.0]`. | -| `test_reward_non_constancy.py` | `server/graders/` | Grader output varies across ≥ 2 synthetic episodes. | -| `test_anti_hivemind_protocol.py` | `cortex/` | 5 protocol steps fire in order; caps enforced (2 rounds, 1 cross-brain challenge, 1 Critic/brain/tick). | -| `test_collapse_detector.py` | `cortex/` | Metacognition flags when all brains recommend the same action. | -| `test_import_graph.py` | repo-wide | No `import server` under `cortex/**`; no `import cortex` under `server/**`; no `import server.simulator` under `training/**`. | -| `test_baselines_smoke.py` | `baselines/` | B1 / B2 / B3 each run one episode on `outbreak_easy`. | -| `test_training_smoke.py` | `training/` | `train_router.main()` runs one episode against a mocked env under 5 s. | - -## Binding rules - -- Every public API in a subsystem's CLAUDE.md has ≥ 1 test here. -- Coverage target: 80% per subsystem; 100% for `server/graders/` and `cortex/anti_hivemind.py`. -- No test may take > 10 s unless marked `@pytest.mark.slow` and gated behind `--runslow`. -- `test_import_graph.py` uses a fresh subprocess import, not `sys.modules` monkey-patching — the latter passes under contamination. - -## Common failure modes - -- Smoke test asserting on current-echo values — breaks when real env logic lands. Assert on shape, not value. -- Module-scope env instantiation in tests — slows collection and hides init errors until runtime. -- Tests that hit the HF Space without a skip guard — CI flakes on rate limits. +# tests/CLAUDE.md + +Test surface per subsystem. Smoke bar first, boundary tests next, coverage last. + +## Belongs here + +- `conftest.py` — repo root on `sys.path` for bare-name imports. +- One test module per subsystem boundary (table below). + +## Does not belong here + +Helpers that mutate real graders, simulator state, or disk. Fixtures that hit the live HF Space — mock or `pytest.skip`. + +## Run commands + +```bash +uv run python -m pytest tests/ -v # all +uv run python -m pytest tests/test_smoke_env.py::test_reset_returns_valid_observation -v # one +uv run python -m pytest --cov tests/ # coverage +``` + +## Required tests — each maps to exactly one subsystem contract + +| File | Scope | Asserts | +|---|---|---| +| `test_package_exports.py` | wire package | Root `__init__` re-exports `CrisisworldcortexAction/Observation/Env`. | +| `test_smoke_env.py` | `server/` env | `reset()` / `step()` return a valid `CrisisworldcortexObservation`. | +| `test_actions_round_trip.py` | `server/` env | 6 MVP outer actions round-trip; `public_communication` is rejected at runtime. | +| `test_reward_shape.py` | `server/graders/` | Every grader returns values in `[0.0, 1.0]`. | +| `test_reward_non_constancy.py` | `server/graders/` | Grader output varies across ≥ 2 synthetic episodes. | +| `test_anti_hivemind_protocol.py` | `cortex/` | 5 protocol steps fire in order; caps enforced (2 rounds, 1 cross-brain challenge, 1 Critic/brain/tick). | +| `test_collapse_detector.py` | `cortex/` | Metacognition flags when all brains recommend the same action. | +| `test_import_graph.py` | repo-wide | No `import server` under `cortex/**`; no `import cortex` under `server/**`; no `import server.simulator` under `training/**`. | +| `test_baselines_smoke.py` | `baselines/` | B1 / B2 / B3 each run one episode on `outbreak_easy`. | +| `test_training_smoke.py` | `training/` | `train_router.main()` runs one episode against a mocked env under 5 s. | + +## Binding rules + +- Every public API in a subsystem's CLAUDE.md has ≥ 1 test here. +- Coverage target: 80% per subsystem; 100% for `server/graders/` and `cortex/anti_hivemind.py`. +- No test may take > 10 s unless marked `@pytest.mark.slow` and gated behind `--runslow`. +- `test_import_graph.py` uses a fresh subprocess import, not `sys.modules` monkey-patching — the latter passes under contamination. + +## Common failure modes + +- Smoke test asserting on current-echo values — breaks when real env logic lands. Assert on shape, not value. +- Module-scope env instantiation in tests — slows collection and hides init errors until runtime. +- Tests that hit the HF Space without a skip guard — CI flakes on rate limits. diff --git a/tests/_helpers/__init__.py b/tests/_helpers/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..279c1113f03acb9c5941957e3534b99053205336 --- /dev/null +++ b/tests/_helpers/__init__.py @@ -0,0 +1,6 @@ +"""Reusable test helpers for cortex/baselines tests. + +Lives alongside ``tests/`` (not under ``cortex/``) so production code +never imports test doubles. Sessions 9-13 reuse the LLMClient stub +defined here. +""" diff --git a/tests/_helpers/llm_stub.py b/tests/_helpers/llm_stub.py new file mode 100644 index 0000000000000000000000000000000000000000..7241e36f3489ceb65168b52b98857a8831a61543 --- /dev/null +++ b/tests/_helpers/llm_stub.py @@ -0,0 +1,82 @@ +"""LLMClient-level test double. + +Drop-in replacement for ``cortex.llm_client.LLMClient`` for tests that +exercise consumers of the client (subagents, brains, council). Differs +from the SDK-level stub in ``tests/test_llm_client.py``: that one +intercepts the OpenAI SDK; this one intercepts the ``LLMClient.chat`` +surface directly, which is what subagents and harnesses see. + +Reused by sessions 9-13 — do not bury role-specific test logic here. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Dict, List, Optional + +from cortex.llm_client import ChatMessage, ChatResponse + + +@dataclass +class RecordedCall: + """One captured ``chat()`` invocation. Tests assert on these.""" + + caller_id: str + messages: List[ChatMessage] + max_tokens: Optional[int] + temperature: Optional[float] + + +@dataclass +class StubLLMClient: + """Quacks like ``LLMClient`` for the ``chat()`` and counter surface. + + Args: + scripted_responses: Yielded one per ``chat()`` call, in order. + Each entry is the response ``content`` string. + prompt_tokens_per_call: Fake prompt-token billing per call. + completion_tokens_per_call: Fake completion-token billing per call. + """ + + scripted_responses: List[str] + prompt_tokens_per_call: int = 50 + completion_tokens_per_call: int = 30 + calls: List[RecordedCall] = field(default_factory=list) + _counters: Dict[str, int] = field(default_factory=dict) + + def chat( + self, + caller_id: str, + messages: List[ChatMessage], + max_tokens: Optional[int] = None, + temperature: Optional[float] = None, + ) -> ChatResponse: + if not self.scripted_responses: + raise RuntimeError( + f"StubLLMClient exhausted: caller_id={caller_id!r}; " + f"add more scripted_responses or assert call_count earlier." + ) + content = self.scripted_responses.pop(0) + self.calls.append( + RecordedCall( + caller_id=caller_id, + messages=list(messages), + max_tokens=max_tokens, + temperature=temperature, + ) + ) + billed = self.prompt_tokens_per_call + self.completion_tokens_per_call + self._counters[caller_id] = self._counters.get(caller_id, 0) + billed + return ChatResponse( + content=content, + finish_reason="stop", + prompt_tokens=self.prompt_tokens_per_call, + completion_tokens=self.completion_tokens_per_call, + ) + + def tokens_used_for(self, caller_id: str) -> int: + return self._counters.get(caller_id, 0) + + @property + def call_count(self) -> int: + return len(self.calls) diff --git a/tests/test_baseline_b1.py b/tests/test_baseline_b1.py index ee0dd498ee326ea282e207de25e9d396fc613448..24d1aefdd7e12bd5688da5b7b7f790b2041867c6 100644 --- a/tests/test_baseline_b1.py +++ b/tests/test_baseline_b1.py @@ -218,7 +218,13 @@ def test_b1_runs_episode_with_valid_json() -> None: def test_b1_parse_failure_submits_synthetic_rejection() -> None: """When the LLM emits unparseable text, B1 submits a synthetic PublicCommunication so the env rejects with accepted=False — landing - r_policy=0 in outer_reward, and the action log shows the rejection. + r_policy=-1.0 in outer_reward (Phase-1 fix per design §19) and the + action log shows the rejection. + + Phase-1 contract: parse-failure now TERMINATES the episode at the + rejection tick (state.terminal = "failure" → obs.done = True). So + only the first parse-failure lands; the second LLM response in the + stub queue never gets dispatched. """ env_inner = CrisisworldcortexEnvironment() env = _InProcessEnvAdapter(env_inner) @@ -233,36 +239,26 @@ def test_b1_parse_failure_submits_synthetic_rejection() -> None: trajectory = agent.run_episode(task="outbreak_easy", seed=0, max_ticks=5) - # Both parse failures were detected and counted. - assert trajectory["parse_failure_count"] == 2 + # Phase-1: parse-failure terminates → only the first marker lands. + assert trajectory["parse_failure_count"] == 1 - # B1 did NOT crash on parse failure — at least 3 ticks ran, even - # though the env may then have hit a terminal (success-on-3-safe-ticks - # or otherwise). What's binding: parse failure does not raise. - assert trajectory["steps_taken"] >= 3, ( + # B1 did NOT crash on parse failure — exactly 1 tick ran (parse-failure + # marker submitted, env terminated episode). + assert trajectory["steps_taken"] == 1, ( f"steps_taken={trajectory['steps_taken']!r} - parse failure " - f"shouldn't kill the agent before tick 3" + f"should terminate at tick 1 under the §19 contract" ) - # The first two action-log entries show V2 rejection (synthetic - # public_communication was submitted; env returned accepted=False). + # The action-log entry shows synthetic public_communication + # (parse-failure marker, honesty=0.0) — rejected by env. log = env_inner._world_state.recent_action_log - assert len(log) >= 2 + assert len(log) == 1 assert log[0].action.kind == "public_communication" assert log[0].accepted is False, "parse-failure synthetic must be rejected by env" - assert log[1].action.kind == "public_communication" - assert log[1].accepted is False - - # The third entry should be the first parsed NoOp. - assert log[2].action.kind == "no_op" - assert log[2].accepted is True - # B1's local trajectory carries the raw snippets for forensic use. + # B1's local trajectory carries the raw snippet for forensic use. assert trajectory["action_history"][0]["parse_failure"] is True assert trajectory["action_history"][0]["raw_llm"] == "I cannot help with that." - assert trajectory["action_history"][1]["parse_failure"] is True - assert trajectory["action_history"][1]["raw_llm"] == "Sorry, no JSON." - assert trajectory["action_history"][2]["parse_failure"] is False def test_b1_caller_id_format_short_colon_separated() -> None: @@ -335,7 +331,10 @@ def test_b1_step_event_carries_rich_context() -> None: from baselines.flat_agent import B1StepEvent env = _InProcessEnvAdapter(CrisisworldcortexEnvironment()) - llm = _StubLLMClient(["I cannot help with that."] + ['{"kind": "no_op"}'] * 5) + # Use a clean-parse first response; parse-failure now terminates the + # episode at tick 1 under the §19 contract, so we exercise the + # tick-1 + tick-2 sequence with both responses being valid JSON. + llm = _StubLLMClient(['{"kind": "no_op"}'] * 5) agent = B1FlatAgent(env=env, llm=llm) events: list[B1StepEvent] = [] @@ -348,18 +347,17 @@ def test_b1_step_event_carries_rich_context() -> None: assert len(events) >= 2 - # Tick 1: parse failure. Submitted action is the synthetic V2-rejected - # PublicCommunication marker; env returns accepted=False; reward in [0,1]. + # Tick 1: clean parse. Submitted is NoOp. error must be None. e1 = events[0] assert e1.tick == 1 - assert e1.parse_failure is True - assert e1.error == "parse_failure" - assert e1.raw_llm == "I cannot help with that." - assert e1.action.kind == "public_communication" - assert 0.0 <= e1.reward <= 1.0 + assert e1.parse_failure is False + assert e1.error is None + assert e1.action.kind == "no_op" + assert e1.raw_llm == '{"kind": "no_op"}' + assert -1.0 <= e1.reward <= 1.0 # Phase-1 range relaxed from [0,1]. assert isinstance(e1.done, bool) - # Tick 2: clean parse. Submitted is NoOp. error must be None. + # Tick 2: another clean parse. Submitted is NoOp. error must be None. e2 = events[1] assert e2.tick == 2 assert e2.parse_failure is False diff --git a/tests/test_cortex_brain_executive.py b/tests/test_cortex_brain_executive.py new file mode 100644 index 0000000000000000000000000000000000000000..e52c6f75f1260eb9db6b390809cb69bfd1c65c3c --- /dev/null +++ b/tests/test_cortex_brain_executive.py @@ -0,0 +1,206 @@ +"""Session 11 - Brain Executive aggregation tests. + +Per Phase A docs/CORTEX_ARCHITECTURE.md Decisions 15-21 + M-FR-3 +(partial evidence union; CandidatePlan and CriticReport schemas have +no evidence field). +""" + +from __future__ import annotations + +import pytest + +from cortex.brains import aggregate_brain_outputs +from cortex.schemas import ( + BeliefState, + CandidatePlan, + CriticReport, + EvidenceCitation, + Hypothesis, + PerceptionReport, + RegionBeliefEstimate, +) +from CrisisWorldCortex.models import ( + DeployResource, + NoOp, + RestrictMovement, +) + + +def _belief(uncertainty: float = 0.4, evidence_count: int = 1) -> BeliefState: + return BeliefState( + brain="epidemiology", + latent_estimates={ + "R1": RegionBeliefEstimate( + estimated_infection_rate=0.05, + estimated_r_effective=1.2, + estimated_compliance=0.85, + ), + }, + hypotheses=[Hypothesis(label="h1", weight=0.6, explanation="rising")], + uncertainty=uncertainty, + reducible_by_more_thought=0.3, + evidence=[ + EvidenceCitation(source="telemetry", ref=f"R1.cases@{i}", excerpt=f"e{i}") + for i in range(evidence_count) + ], + ) + + +def _plan(action=None, confidence: float = 0.75, expected_value: float = 0.6) -> CandidatePlan: + if action is None: + action = DeployResource(region="R1", resource_type="test_kits", quantity=100) + return CandidatePlan( + action_sketch="Deploy 100 test_kits to R1", + expected_outer_action=action, + expected_value=expected_value, + cost=200.0, + assumptions=["kits available"], + falsifiers=["R1 cases drop without intervention"], + confidence=confidence, + ) + + +def _critic(severity: float = 0.3) -> CriticReport: + return CriticReport( + brain="epidemiology", + target_plan_id="plan-0", + attacks=["limited reach"], + missing_considerations=[], + would_change_mind_if=[], + severity=severity, + ) + + +def _perception(evidence_count: int = 1) -> PerceptionReport: + return PerceptionReport( + brain="epidemiology", + salient_signals=["R1 cases rising"], + anomalies=[], + confidence=0.7, + evidence=[ + EvidenceCitation(source="telemetry", ref=f"R1.perception@{i}", excerpt=f"p{i}") + for i in range(evidence_count) + ], + ) + + +# T4 +def test_brain_executive_aggregates_subagent_outputs() -> None: + rec = aggregate_brain_outputs( + brain_id="epidemiology", + perception=_perception(), + beliefs=[_belief()], + plans=[_plan()], + critics=[_critic()], + ) + assert rec.brain == "epidemiology" + assert rec.top_action.kind == "deploy_resource" + assert rec.top_confidence > 0.0 + assert rec.tokens_used == 0 + + +# T5 -- Decision 16 +def test_brain_executive_top_confidence_includes_uncertainty() -> None: + rec = aggregate_brain_outputs( + brain_id="epidemiology", + perception=_perception(), + beliefs=[_belief(uncertainty=0.4)], + plans=[_plan(confidence=0.75)], + critics=[_critic()], + ) + # D16: top_confidence == confidence x (1 - uncertainty) == 0.75 x 0.6 == 0.45 + assert rec.top_confidence == pytest.approx(0.45) + + +# T6 -- Decision 17 +def test_brain_executive_minority_actions_excludes_top() -> None: + plan_a = _plan( + action=DeployResource(region="R1", resource_type="test_kits", quantity=100), + confidence=0.8, + expected_value=0.7, + ) + plan_b = _plan( + action=RestrictMovement(region="R1", severity="moderate"), + confidence=0.5, + expected_value=0.4, + ) + # plan_a wins: 0.8 * 0.7 = 0.56 > 0.5 * 0.4 = 0.20 + + rec = aggregate_brain_outputs( + brain_id="epidemiology", + perception=_perception(), + beliefs=[_belief(), _belief(uncertainty=0.5)], + plans=[plan_a, plan_b], + critics=[_critic(), _critic()], + ) + + assert rec.top_action.kind == "deploy_resource" + assert len(rec.minority_actions) == 1 + assert rec.minority_actions[0].kind == "restrict_movement" + + +# T7 -- Decision 20 + M-FR-3 +def test_brain_executive_evidence_union() -> None: + perception = _perception(evidence_count=1) + belief = _belief(evidence_count=2) + + rec = aggregate_brain_outputs( + brain_id="epidemiology", + perception=perception, + beliefs=[belief], + plans=[_plan()], + critics=[_critic()], + ) + + # M-FR-3: union of perception.evidence + belief.evidence (3 total) + assert len(rec.evidence) == 3 + assert rec.evidence[0].ref.startswith("R1.perception") + assert rec.evidence[1].ref.startswith("R1.cases") + + +# T8 -- empty fallback +def test_brain_executive_handles_empty_subagent_outputs() -> None: + empty_belief = BeliefState( + brain="epidemiology", + latent_estimates={}, + hypotheses=[], + uncertainty=1.0, + reducible_by_more_thought=0.0, + evidence=[], + ) + empty_plan = CandidatePlan( + action_sketch="(empty)", + expected_outer_action=NoOp(), + expected_value=0.0, + cost=0.0, + assumptions=[], + falsifiers=[], + confidence=0.0, + ) + empty_critic = CriticReport( + brain="epidemiology", + target_plan_id="", + attacks=[], + missing_considerations=[], + would_change_mind_if=[], + severity=0.0, + ) + empty_perception = PerceptionReport( + brain="epidemiology", + salient_signals=[], + anomalies=[], + confidence=0.0, + evidence=[], + ) + + rec = aggregate_brain_outputs( + brain_id="epidemiology", + perception=empty_perception, + beliefs=[empty_belief], + plans=[empty_plan], + critics=[empty_critic], + ) + + assert rec.top_action.kind == "no_op" + assert rec.top_confidence == 0.0 + assert rec.uncertainty == 1.0 diff --git a/tests/test_cortex_brain_smoke.py b/tests/test_cortex_brain_smoke.py new file mode 100644 index 0000000000000000000000000000000000000000..3e6fb6a71894e3400ac664892dff1b7f677b1559 --- /dev/null +++ b/tests/test_cortex_brain_smoke.py @@ -0,0 +1,142 @@ +"""Session 11 - Single-brain end-to-end smoke + no-LLM-in-Python-layers test. + +T9 asserts Brain.compute_perception and Brain.compute_lens are pure +Python (zero LLM calls). T10 is the integration smoke gate per Phase A +section 10: a single brain runs end-to-end on a real observation, +returns a BrainRecommendation. Three LLM calls in canonical +WorldModeler -> Planner -> Critic order with locked caller_id format. +""" + +from __future__ import annotations + +import json + +from cortex.brains import EpiBrain +from cortex.schemas import BrainRecommendation +from CrisisWorldCortex.models import ( + CrisisworldcortexObservation, + RegionTelemetry, + ResourceInventory, +) +from tests._helpers.llm_stub import StubLLMClient + + +def _make_obs() -> CrisisworldcortexObservation: + return CrisisworldcortexObservation( + regions=[ + RegionTelemetry( + region=f"R{i + 1}", + reported_cases_d_ago=5 if i == 0 else 1, + hospital_load=0.3 if i == 0 else 0.1, + compliance_proxy=0.85, + ) + for i in range(4) + ], + resources=ResourceInventory( + test_kits=1000, + hospital_beds_free=500, + mobile_units=20, + vaccine_doses=2000, + ), + active_restrictions=[], + legal_constraints=[], + tick=3, + ticks_remaining=9, + cognition_budget_remaining=5200, + recent_action_log=[], + ) + + +_VALID_BELIEF = json.dumps( + { + "brain": "epidemiology", + "latent_estimates": { + "R1": { + "estimated_infection_rate": 0.05, + "estimated_r_effective": 1.2, + "estimated_compliance": 0.85, + "confidence_intervals": {}, + } + }, + "hypotheses": [{"label": "rising", "weight": 0.6, "explanation": "R1 cases up"}], + "uncertainty": 0.4, + "reducible_by_more_thought": 0.3, + "evidence": [ + {"source": "telemetry", "ref": "R1.cases", "excerpt": "5"}, + {"source": "policy", "ref": "R1.restriction", "excerpt": "none"}, + ], + } +) + +_VALID_PLAN = json.dumps( + { + "action_sketch": "Deploy 100 test_kits to R1", + "expected_outer_action": { + "kind": "deploy_resource", + "region": "R1", + "resource_type": "test_kits", + "quantity": 100, + }, + "expected_value": 0.6, + "cost": 200.0, + "assumptions": ["kits inventory > 100"], + "falsifiers": ["R1 cases drop without intervention"], + "confidence": 0.75, + } +) + +_VALID_CRITIC = json.dumps( + { + "brain": "epidemiology", + "target_plan_id": "plan-0", + "attacks": ["ignores R3 hospital saturation"], + "missing_considerations": [], + "would_change_mind_if": [], + "severity": 0.3, + } +) + + +# T9 +def test_brain_runs_zero_llm_calls_in_python_layers() -> None: + """Perception and Lens are pure Python; no LLMClient invocation.""" + stub = StubLLMClient(scripted_responses=[]) # any chat() would raise + brain = EpiBrain(stub) + obs = _make_obs() + + perception = brain.compute_perception(obs) + lensed = brain.compute_lens(obs, last_reward=0.0) + + assert stub.call_count == 0 + assert perception.brain == "epidemiology" + assert lensed.brain == "epidemiology" + + +# T10 -- integration smoke gate (Phase A section 10) +def test_brain_smoke_one_tick_three_llm_calls_in_order() -> None: + """Full round-1 tick: WorldModeler -> Planner -> Critic, in that order.""" + stub = StubLLMClient(scripted_responses=[_VALID_BELIEF, _VALID_PLAN, _VALID_CRITIC]) + brain = EpiBrain(stub) + obs = _make_obs() + + rec = brain.run_tick(obs, last_reward=0.0, tick=3) + + assert isinstance(rec, BrainRecommendation) + assert rec.brain == "epidemiology" + assert rec.top_action.kind == "deploy_resource" + assert stub.call_count == 3, "exactly 3 LLM calls per Phase A 'WM + Planner + Critic'" + + # Order pin per user adjustment: WM (s0) -> Planner (s1) -> Critic (s2) + assert stub.calls[0].caller_id.endswith(":world_modeler:t3:r1:s0"), ( + f"first call must be WorldModeler, got {stub.calls[0].caller_id!r}" + ) + assert stub.calls[1].caller_id.endswith(":planner:t3:r1:s1"), ( + f"second call must be Planner, got {stub.calls[1].caller_id!r}" + ) + assert stub.calls[2].caller_id.endswith(":critic:t3:r1:s2"), ( + f"third call must be Critic, got {stub.calls[2].caller_id!r}" + ) + + # Brain prefix locks + for call in stub.calls: + assert call.caller_id.startswith("cortex:epidemiology:") diff --git a/tests/test_cortex_lenses.py b/tests/test_cortex_lenses.py new file mode 100644 index 0000000000000000000000000000000000000000..952349225779bfe163abed60757afe6e21031d70 --- /dev/null +++ b/tests/test_cortex_lenses.py @@ -0,0 +1,238 @@ +"""Session 10 - Cortex lens tests. + +Per Phase A docs/CORTEX_ARCHITECTURE.md Decisions 9-14 + §2 A1 and the +user's Session 10 proposal acceptance with 8 tests + the M-FR-4 rename +(epi_pressure) and T7 tightening (non-bool float). +""" + +from __future__ import annotations + +from typing import Iterable + +import pytest + +from cortex.lenses import lens_for +from cortex.schemas import BrainLensedObservation +from CrisisWorldCortex.models import ( + CrisisworldcortexObservation, + Escalate, + ExecutedAction, + LegalConstraint, + NoOp, + RegionTelemetry, + ResourceInventory, + Restriction, +) + +# ============================================================================ +# Test fixtures +# ============================================================================ + + +def _make_obs( + cases_per_region: Iterable[int] = (5, 1, 1, 1), + hospital_loads: Iterable[float] = (0.3, 0.1, 0.1, 0.1), + compliance_proxies: Iterable[float] = (0.85, 0.95, 0.95, 0.95), + test_kits: int = 1000, + hospital_beds_free: int = 500, + mobile_units: int = 20, + vaccine_doses: int = 2000, + restrictions: Iterable[Restriction] = (), + legal_constraints: Iterable[LegalConstraint] = (), + recent_action_log: Iterable[ExecutedAction] = (), +) -> CrisisworldcortexObservation: + cases = list(cases_per_region) + loads = list(hospital_loads) + comps = list(compliance_proxies) + return CrisisworldcortexObservation( + regions=[ + RegionTelemetry( + region=f"R{i + 1}", + reported_cases_d_ago=cases[i], + hospital_load=loads[i], + compliance_proxy=comps[i], + ) + for i in range(4) + ], + resources=ResourceInventory( + test_kits=test_kits, + hospital_beds_free=hospital_beds_free, + mobile_units=mobile_units, + vaccine_doses=vaccine_doses, + ), + active_restrictions=list(restrictions), + legal_constraints=list(legal_constraints), + tick=3, + ticks_remaining=9, + cognition_budget_remaining=5200, + recent_action_log=list(recent_action_log), + ) + + +# ============================================================================ +# T1 - Epi lens emphasizes telemetry; uses epi_pressure (M-FR-4 rename) +# ============================================================================ + + +def test_epi_lens_emphasizes_telemetry() -> None: + obs = _make_obs() + lensed = lens_for("epidemiology", obs, last_reward=0.5) + + assert isinstance(lensed, BrainLensedObservation) + assert lensed.brain == "epidemiology" + assert lensed.last_reward == 0.5 + + keys = set(lensed.derived_features.keys()) + assert {"epi_pressure", "worst_region_infection", "transmission_rate_trend"} <= keys + + # M-FR-2: trend is 0.0 in MVP (no history available in single-obs lens) + assert lensed.derived_features["transmission_rate_trend"] == 0.0 + + assert "regions[*].reported_cases_d_ago" in lensed.salient_field_ids + assert "regions[*].hospital_load" in lensed.salient_field_ids + + +# ============================================================================ +# T2 - Logistics lens emphasizes resources +# ============================================================================ + + +def test_logistics_lens_emphasizes_resources() -> None: + obs = _make_obs(test_kits=100, hospital_beds_free=50, mobile_units=10, vaccine_doses=200) + lensed = lens_for("logistics", obs, last_reward=0.5) + + assert lensed.brain == "logistics" + keys = set(lensed.derived_features.keys()) + expected_keys = { + "total_inventory", + "hospital_load_max", + "deployment_feasibility_R1", + "deployment_feasibility_R2", + "deployment_feasibility_R3", + "deployment_feasibility_R4", + } + assert expected_keys <= keys + + # 100 + 50 + 10 + 200 = 360 + assert lensed.derived_features["total_inventory"] == 360.0 + + assert "resources.test_kits" in lensed.salient_field_ids + + +# ============================================================================ +# T3 - Governance lens emphasizes legal +# ============================================================================ + + +def test_governance_lens_emphasizes_legal() -> None: + obs = _make_obs( + restrictions=[Restriction(region="R1", severity="moderate", ticks_remaining=3)], + legal_constraints=[ + LegalConstraint(rule_id="L1", blocked_action="restrict_movement.strict") + ], + ) + lensed = lens_for("governance", obs, last_reward=0.5) + + assert lensed.brain == "governance" + keys = set(lensed.derived_features.keys()) + assert { + "escalation_unlocked_strict", + "legal_constraints_count", + "restrictions_active_count", + } <= keys + + assert lensed.derived_features["legal_constraints_count"] == 1.0 + assert lensed.derived_features["restrictions_active_count"] == 1.0 + assert "active_restrictions[*]" in lensed.salient_field_ids + + +# ============================================================================ +# T4 - Lens does NOT strip raw_obs (D13) +# ============================================================================ + + +def test_lens_does_not_strip_raw_obs() -> None: + obs = _make_obs(restrictions=[Restriction(region="R1", severity="moderate", ticks_remaining=3)]) + + for brain in ("epidemiology", "logistics", "governance"): + lensed = lens_for(brain, obs, last_reward=0.0) + # Pydantic deep-equality on the full observation + assert lensed.raw_obs == obs + + +# ============================================================================ +# T5 - V2 brain ids raise KeyError (Decision 9, post-review) +# ============================================================================ + + +def test_lens_for_v2_brain_raises_key_error() -> None: + obs = _make_obs() + + for v2_brain in ("communications", "equity"): + with pytest.raises(KeyError): + lens_for(v2_brain, obs, last_reward=0.0) + + with pytest.raises(KeyError): + lens_for("not_a_brain", obs, last_reward=0.0) + + +# ============================================================================ +# T7 - All derived_features values are non-bool floats (D14 + tightened) +# ============================================================================ + + +def test_lens_derived_features_all_floats() -> None: + obs = _make_obs( + restrictions=[Restriction(region="R1", severity="moderate", ticks_remaining=3)], + legal_constraints=[ + LegalConstraint(rule_id="L1", blocked_action="restrict_movement.strict") + ], + ) + + for brain in ("epidemiology", "logistics", "governance"): + lensed = lens_for(brain, obs, last_reward=0.0) + for key, value in lensed.derived_features.items(): + assert isinstance(value, float) and not isinstance(value, bool), ( + f"derived_features[{key!r}] = {value!r} ({type(value).__name__}) " + f"is not a non-bool float" + ) + + +# ============================================================================ +# T8 - Governance lens detects accepted escalate(national) +# ============================================================================ + + +def test_governance_lens_detects_escalation_unlocked() -> None: + obs_with_accepted = _make_obs( + recent_action_log=[ + ExecutedAction(tick=1, action=NoOp(), accepted=True), + ExecutedAction(tick=2, action=Escalate(to_authority="national"), accepted=True), + ] + ) + lensed = lens_for("governance", obs_with_accepted, last_reward=0.0) + assert lensed.derived_features["escalation_unlocked_strict"] == 1.0 + + obs_without = _make_obs( + recent_action_log=[ExecutedAction(tick=1, action=NoOp(), accepted=True)] + ) + lensed_w = lens_for("governance", obs_without, last_reward=0.0) + assert lensed_w.derived_features["escalation_unlocked_strict"] == 0.0 + + # Rejected escalate must NOT count + obs_rejected = _make_obs( + recent_action_log=[ + ExecutedAction(tick=1, action=Escalate(to_authority="national"), accepted=False), + ] + ) + lensed_r = lens_for("governance", obs_rejected, last_reward=0.0) + assert lensed_r.derived_features["escalation_unlocked_strict"] == 0.0 + + # Accepted escalate(regional) must NOT count -- only "national" unlocks strict + obs_regional = _make_obs( + recent_action_log=[ + ExecutedAction(tick=1, action=Escalate(to_authority="regional"), accepted=True), + ] + ) + lensed_re = lens_for("governance", obs_regional, last_reward=0.0) + assert lensed_re.derived_features["escalation_unlocked_strict"] == 0.0 diff --git a/tests/test_cortex_perception.py b/tests/test_cortex_perception.py new file mode 100644 index 0000000000000000000000000000000000000000..0f6fc0553686ff19e38cd292b951667e725a0775 --- /dev/null +++ b/tests/test_cortex_perception.py @@ -0,0 +1,116 @@ +"""Session 11 - Perception subagent tests (deterministic Python; no LLM). + +Per cortex/CLAUDE.md binding (Perception is pure Python) and Phase A +Decisions 9 (V2 KeyError) + 63 (salient_signals cap at 5). +""" + +from __future__ import annotations + +from typing import Iterable + +import pytest + +from cortex.schemas import PerceptionReport +from cortex.subagents import perception_for +from CrisisWorldCortex.models import ( + CrisisworldcortexObservation, + LegalConstraint, + RegionTelemetry, + ResourceInventory, + Restriction, +) + + +def _make_obs( + cases_per_region: Iterable[int] = (5, 1, 0, 0), + hospital_loads: Iterable[float] = (0.7, 0.1, 0.1, 0.1), + test_kits: int = 200, + hospital_beds_free: int = 50, + mobile_units: int = 2, + vaccine_doses: int = 300, + restrictions: Iterable[Restriction] = (), + legal_constraints: Iterable[LegalConstraint] = (), +) -> CrisisworldcortexObservation: + cases = list(cases_per_region) + loads = list(hospital_loads) + return CrisisworldcortexObservation( + regions=[ + RegionTelemetry( + region=f"R{i + 1}", + reported_cases_d_ago=cases[i], + hospital_load=loads[i], + compliance_proxy=0.85, + ) + for i in range(4) + ], + resources=ResourceInventory( + test_kits=test_kits, + hospital_beds_free=hospital_beds_free, + mobile_units=mobile_units, + vaccine_doses=vaccine_doses, + ), + active_restrictions=list(restrictions), + legal_constraints=list(legal_constraints), + tick=3, + ticks_remaining=9, + cognition_budget_remaining=5200, + recent_action_log=[], + ) + + +# T1 +def test_perception_runs_without_llm_call() -> None: + """Perception is pure Python; the function does not take an LLMClient.""" + obs = _make_obs() + for brain in ("epidemiology", "logistics", "governance"): + report = perception_for(brain, obs) + assert isinstance(report, PerceptionReport) + assert report.brain == brain + assert isinstance(report.confidence, float) + assert 0.0 <= report.confidence <= 1.0 + + +# T2 +def test_perception_for_v2_brain_raises_key_error() -> None: + obs = _make_obs() + for v2_brain in ("communications", "equity"): + with pytest.raises(KeyError): + perception_for(v2_brain, obs) + with pytest.raises(KeyError): + perception_for("not_a_brain", obs) + + +# T3 +@pytest.mark.parametrize("brain", ["epidemiology", "logistics", "governance"]) +def test_perception_brain_specific_signals(brain: str) -> None: + obs = _make_obs( + cases_per_region=(20, 1, 0, 0), + hospital_loads=(0.7, 0.1, 0.1, 0.1), + test_kits=100, # below threshold (300) + hospital_beds_free=50, # below threshold (100) + mobile_units=2, # below threshold (5) + vaccine_doses=200, # below threshold (500) + restrictions=[ + Restriction(region="R1", severity="moderate", ticks_remaining=3), + ], + legal_constraints=[ + LegalConstraint(rule_id="L1", blocked_action="restrict_movement.strict"), + ], + ) + report = perception_for(brain, obs) + + assert report.brain == brain + # Decision 63 / OQ-2 cap: at most 5 entries + assert len(report.salient_signals) <= 5 + + if brain == "epidemiology": + assert any("R1" in s for s in report.salient_signals), ( + f"epi salient_signals should reference R1, got {report.salient_signals}" + ) + elif brain == "logistics": + joined = " ".join(report.salient_signals).lower() + assert "kits" in joined or "mobile" in joined or "vaccine" in joined or "beds" in joined + elif brain == "governance": + assert any("R1" in s and "moderate" in s.lower() for s in report.salient_signals), ( + f"governance salient_signals should mention R1 moderate, got {report.salient_signals}" + ) diff --git a/tests/test_cortex_subagents.py b/tests/test_cortex_subagents.py new file mode 100644 index 0000000000000000000000000000000000000000..115f5caaff660d67fb366d4961f14c35cbc02b9f --- /dev/null +++ b/tests/test_cortex_subagents.py @@ -0,0 +1,387 @@ +"""Session 9 - Cortex subagent tests (WorldModeler, Planner, Critic). + +Per Phase A docs/CORTEX_ARCHITECTURE.md Decisions 1-8 + 62 and the user's +proposal acceptance with 11 tests total. RED-tests-first. +""" + +from __future__ import annotations + +import json +import re +from typing import Any, Dict, Optional + +import pytest + +from cortex.schemas import ( + BeliefState, + CandidatePlan, + CriticReport, + EvidenceCitation, + PerceptionReport, + SubagentInput, +) +from cortex.subagents import ( + PROMPTS_DIR, + CriticSubagent, + PlannerSubagent, + WorldModelerSubagent, +) +from tests._helpers.llm_stub import StubLLMClient + +# ============================================================================ +# Test fixtures +# ============================================================================ + + +_VALID_BELIEF_PAYLOAD: Dict[str, Any] = { + "brain": "epidemiology", + "latent_estimates": { + "R1": { + "estimated_infection_rate": 0.05, + "estimated_r_effective": 1.2, + "estimated_compliance": 0.85, + "confidence_intervals": {}, + }, + }, + "hypotheses": [{"label": "rising-r1", "weight": 0.6, "explanation": "telemetry trending up"}], + "uncertainty": 0.4, + "reducible_by_more_thought": 0.3, + "evidence": [ + {"source": "telemetry", "ref": "R1.reported_cases@t3", "excerpt": "rising"}, + {"source": "policy", "ref": "R1.restriction", "excerpt": "moderate"}, + ], +} + + +_VALID_PLAN_PAYLOAD: Dict[str, Any] = { + "action_sketch": "Deploy 100 test_kits to R1", + "expected_outer_action": { + "kind": "deploy_resource", + "region": "R1", + "resource_type": "test_kits", + "quantity": 100, + }, + "expected_value": 0.6, + "cost": 200.0, + "assumptions": ["kits inventory > 100"], + "falsifiers": ["R1 cases drop without intervention"], + "confidence": 0.75, +} + + +_VALID_CRITIC_PAYLOAD: Dict[str, Any] = { + "brain": "logistics", + "target_plan_id": "plan-1", + "attacks": ["ignores R3 hospital saturation"], + "missing_considerations": ["compliance decay over 4 ticks"], + "would_change_mind_if": ["new R3 telemetry shows under-utilisation"], + "severity": 0.6, +} + + +def _valid_json_for(role: str, brain: str = "epidemiology") -> str: + """Return a valid JSON-schema response for ``role``, brain-substituted.""" + if role == "world_modeler": + payload = dict(_VALID_BELIEF_PAYLOAD) + payload["brain"] = brain + return json.dumps(payload) + if role == "planner": + return json.dumps(_VALID_PLAN_PAYLOAD) + if role == "critic": + payload = dict(_VALID_CRITIC_PAYLOAD) + payload["brain"] = brain + return json.dumps(payload) + raise ValueError(f"unknown role: {role}") + + +def _make_subagent_input( + brain: str = "epidemiology", + role: str = "world_modeler", + tick: int = 3, + round_: int = 1, + target_plan_id: Optional[str] = None, +) -> SubagentInput: + """Minimal valid SubagentInput. ``round_`` arg avoids shadowing builtin.""" + return SubagentInput( + brain=brain, + role=role, + tick=tick, + round=round_, + perception=PerceptionReport( + brain=brain, + salient_signals=["R1 cases rising"], + anomalies=[], + confidence=0.7, + evidence=[EvidenceCitation(source="telemetry", ref="R1.cases", excerpt="rising")], + ), + prior_belief=None, + prior_plans=[], + target_plan_id=target_plan_id, + last_reward=0.5, + recent_action_log_excerpt=[], + ) + + +# ============================================================================ +# T1 - WorldModeler emits BeliefState +# ============================================================================ + + +def test_world_modeler_emits_belief_state() -> None: + stub = StubLLMClient(scripted_responses=[_valid_json_for("world_modeler", "epidemiology")]) + agent = WorldModelerSubagent(llm_client=stub) + input_data = _make_subagent_input(brain="epidemiology", role="world_modeler") + + result = agent.run(input_data, step_idx=0) + + assert isinstance(result, BeliefState) + assert result.brain == "epidemiology" + assert "R1" in result.latent_estimates + assert len(result.evidence) >= 1 + assert stub.call_count == 1 + + +# ============================================================================ +# T2 - Planner emits CandidatePlan +# ============================================================================ + + +def test_planner_emits_candidate_plan() -> None: + stub = StubLLMClient(scripted_responses=[_valid_json_for("planner", "epidemiology")]) + agent = PlannerSubagent(llm_client=stub) + input_data = _make_subagent_input(brain="epidemiology", role="planner") + + result = agent.run(input_data, step_idx=1) + + assert isinstance(result, CandidatePlan) + assert result.expected_outer_action.kind == "deploy_resource" + assert result.confidence == 0.75 + + +# ============================================================================ +# T3 - Critic emits CriticReport +# ============================================================================ + + +def test_critic_emits_critic_report() -> None: + stub = StubLLMClient(scripted_responses=[_valid_json_for("critic", "logistics")]) + agent = CriticSubagent(llm_client=stub) + input_data = _make_subagent_input(brain="logistics", role="critic", target_plan_id="plan-1") + + result = agent.run(input_data, step_idx=2) + + assert isinstance(result, CriticReport) + assert result.brain == "logistics" + assert result.target_plan_id == "plan-1" + assert result.severity == 0.6 + + +# ============================================================================ +# T4 - Parse failure then retry succeeds (2 LLM calls) +# ============================================================================ + + +def test_subagent_parse_failure_then_retry_succeeds() -> None: + stub = StubLLMClient( + scripted_responses=["not-json-garbage", _valid_json_for("world_modeler", "epidemiology")] + ) + agent = WorldModelerSubagent(llm_client=stub) + + result = agent.run(_make_subagent_input(), step_idx=0) + + assert isinstance(result, BeliefState) + assert result.brain == "epidemiology" + assert stub.call_count == 2, "expected 1 initial call + 1 retry" + + +# ============================================================================ +# T5 - Parse failure twice -> empty fallback (no third LLM call) +# ============================================================================ + + +def test_subagent_parse_failure_then_retry_fails_returns_empty() -> None: + stub = StubLLMClient(scripted_responses=["garbage-1", "garbage-2"]) + agent = WorldModelerSubagent(llm_client=stub) + + result = agent.run(_make_subagent_input(), step_idx=0) + + assert isinstance(result, BeliefState) + assert result.brain == "epidemiology" + assert result.latent_estimates == {} + assert result.hypotheses == [] + assert result.evidence == [] + assert result.uncertainty == 1.0 + assert result.reducible_by_more_thought == 0.0 + assert stub.call_count == 2, "must NOT make a 3rd call after retry failure" + + +# ============================================================================ +# T6 - caller_id format matches Phase A Decision 7 +# ============================================================================ + + +_CALLER_ID_RE = re.compile( + r"^cortex:(epidemiology|logistics|governance):" + r"(world_modeler|planner|critic):" + r"t\d+:r[12]:s\d+$" +) + + +@pytest.mark.parametrize( + "role_cls,brain,role_name", + [ + (WorldModelerSubagent, "epidemiology", "world_modeler"), + (PlannerSubagent, "logistics", "planner"), + (CriticSubagent, "governance", "critic"), + ], +) +def test_subagent_caller_id_format( + role_cls: type, + brain: str, + role_name: str, +) -> None: + stub = StubLLMClient(scripted_responses=[_valid_json_for(role_name, brain)]) + agent = role_cls(llm_client=stub) + input_data = _make_subagent_input( + brain=brain, + role=role_name, + tick=7, + round_=2, + target_plan_id="plan-X" if role_name == "critic" else None, + ) + + agent.run(input_data, step_idx=4) + + caller_id = stub.calls[0].caller_id + assert _CALLER_ID_RE.match(caller_id), ( + f"caller_id={caller_id!r} does not match the locked format" + ) + assert caller_id == f"cortex:{brain}:{role_name}:t7:r2:s4" + + +# ============================================================================ +# T8 - SYS prompt loaded from prompts/.txt and brain-formatted +# (folds in the prompt-formatting refinement: format() must not raise) +# ============================================================================ + + +@pytest.mark.parametrize( + "role_cls,role_name", + [ + (WorldModelerSubagent, "world_modeler"), + (PlannerSubagent, "planner"), + (CriticSubagent, "critic"), + ], +) +def test_subagent_uses_loaded_prompt_from_file(role_cls: type, role_name: str) -> None: + raw = (PROMPTS_DIR / f"{role_name}.txt").read_text(encoding="utf-8") + + # Refinement: format() must not raise even with extra kwargs ignored. + # Catches {{/}}-escape regressions in JSON-schema sections of the prompts. + formatted = raw.format(brain="epidemiology", target_plan_id="plan-X") + assert isinstance(formatted, str) + assert "epidemiology" in formatted + + stub = StubLLMClient(scripted_responses=[_valid_json_for(role_name, "epidemiology")]) + agent = role_cls(llm_client=stub) + input_data = _make_subagent_input( + brain="epidemiology", + role=role_name, + target_plan_id="plan-X" if role_name == "critic" else None, + ) + + agent.run(input_data, step_idx=0) + + sys_msg = stub.calls[0].messages[0] + assert sys_msg.role == "system" + assert sys_msg.content == formatted + + +# ============================================================================ +# T9 - Token counter is billed to the expected caller_id +# ============================================================================ + + +def test_subagent_token_counter_billed_correctly() -> None: + stub = StubLLMClient(scripted_responses=[_valid_json_for("world_modeler", "epidemiology")]) + agent = WorldModelerSubagent(llm_client=stub) + input_data = _make_subagent_input(brain="epidemiology", role="world_modeler", tick=3, round_=1) + + agent.run(input_data, step_idx=0) + + expected_caller_id = "cortex:epidemiology:world_modeler:t3:r1:s0" + assert stub.tokens_used_for(expected_caller_id) > 0, ( + "tokens must be billed to the per-role caller_id, not silently lost" + ) + assert stub.tokens_used_for("never:called") == 0 + + +# ============================================================================ +# T10 - empty_fallback shape locked per Phase A Decisions 6 + 62 +# ============================================================================ + + +def test_subagent_empty_fallback_shape_locked() -> None: + # WorldModeler: empty BeliefState + bs = WorldModelerSubagent.empty_fallback("epidemiology") + assert isinstance(bs, BeliefState) + assert bs.brain == "epidemiology" + assert bs.latent_estimates == {} + assert bs.hypotheses == [] + assert bs.uncertainty == 1.0 + assert bs.reducible_by_more_thought == 0.0 + assert bs.evidence == [] + + # Planner: empty CandidatePlan with NoOp + confidence=0 + cp = PlannerSubagent.empty_fallback("epidemiology") + assert isinstance(cp, CandidatePlan) + assert cp.expected_outer_action.kind == "no_op" + assert cp.expected_value == 0.0 + assert cp.cost == 0.0 + assert cp.assumptions == [] + assert cp.falsifiers == [] + assert cp.confidence == 0.0 + + # Critic: empty CriticReport with severity=0 + cr = CriticSubagent.empty_fallback("epidemiology", target_plan_id="plan-X") + assert isinstance(cr, CriticReport) + assert cr.brain == "epidemiology" + assert cr.target_plan_id == "plan-X" + assert cr.attacks == [] + assert cr.missing_considerations == [] + assert cr.would_change_mind_if == [] + assert cr.severity == 0.0 + + +# ============================================================================ +# T11 - retry call uses chat-history continuation (sys + usr + bad + retry) +# ============================================================================ + + +def test_subagent_run_uses_chat_history_on_retry() -> None: + stub = StubLLMClient( + scripted_responses=["bad-json", _valid_json_for("world_modeler", "epidemiology")] + ) + agent = WorldModelerSubagent(llm_client=stub) + + agent.run(_make_subagent_input(brain="epidemiology", role="world_modeler"), step_idx=0) + + assert stub.call_count == 2, "expected 2 LLM calls (initial + retry)" + call1, call2 = stub.calls + + # call 1 has the original sys + user (2 messages). + assert len(call1.messages) == 2 + assert call1.messages[0].role == "system" + assert call1.messages[1].role == "user" + + # call 2 must contain: original sys + original user + assistant(bad-json) + retry-user. + # Without chat-history continuation the LLM loses schema context on retry. + assert len(call2.messages) == 4, "retry must append to the chat history, not start fresh" + assert call2.messages[0].role == "system" + assert call2.messages[0].content == call1.messages[0].content + assert call2.messages[1].role == "user" + assert call2.messages[1].content == call1.messages[1].content + assert call2.messages[2].role == "assistant" + assert call2.messages[2].content == "bad-json" + assert call2.messages[3].role == "user" + assert "failed to parse" in call2.messages[3].content.lower() diff --git a/tests/test_outer_reward_in_range.py b/tests/test_outer_reward_in_range.py index 1b59ae0fbccbb05bb2d08ba4b45c604d74de6558..377cce6a9d5770efbb0ffb4bc35ea917d5c36335 100644 --- a/tests/test_outer_reward_in_range.py +++ b/tests/test_outer_reward_in_range.py @@ -1,8 +1,10 @@ -"""Outer reward stays in ``[0.0, 1.0]`` across diverse states. +"""Outer reward stays in ``[-1.0, 1.0]`` across diverse states. -Required by ``server/CLAUDE.md`` ("Every grader scalar reward component -lives in ``[0.0, 1.0]``") and the ``tests/CLAUDE.md`` row -``test_reward_shape.py``. Covers: +Range relaxed by Workstream-B Phase-1 fix (M2-A): rejected actions land +``r_policy = -0.5`` and parse-failure markers land ``r_policy = -1.0``, +so the per-tick total can go negative. Upper bound stays at 1.0. + +Covers: - Initial state (right after ``load_task``, no ticks applied). - Mid-episode rollouts on all 3 tasks with varied actions. @@ -29,7 +31,7 @@ def test_outer_reward_in_range_at_episode_start() -> None: for name in TASKS: state = load_task(name, episode_seed=0) r = outer_reward(state, NoOp()) - assert 0.0 <= r <= 1.0, f"{name}: r={r!r} out of [0,1] at tick 0" + assert -1.0 <= r <= 1.0, f"{name}: r={r!r} out of [-1,1] at tick 0" def test_outer_reward_in_range_during_rollout() -> None: @@ -51,8 +53,8 @@ def test_outer_reward_in_range_during_rollout() -> None: for action in actions: state = apply_tick(state, action) r = outer_reward(state, action) - assert 0.0 <= r <= 1.0, ( - f"{name} tick={state.tick}: r={r!r} out of [0,1] after action kind={action.kind!r}" + assert -1.0 <= r <= 1.0, ( + f"{name} tick={state.tick}: r={r!r} out of [-1,1] after action kind={action.kind!r}" ) @@ -67,7 +69,7 @@ def test_outer_reward_in_range_with_high_infection() -> None: region.S, region.E, region.I, region.R = 0.0, 0.0, 0.95, 0.05 state.tick = state.max_ticks # r_time → 0 r = outer_reward(state, NoOp()) - assert 0.0 <= r <= 1.0, f"high-I worst case: r={r!r}" + assert -1.0 <= r <= 1.0, f"high-I worst case: r={r!r}" def test_outer_reward_in_range_with_rejected_actions() -> None: @@ -81,7 +83,7 @@ def test_outer_reward_in_range_with_rejected_actions() -> None: ) state = apply_tick(state, a_v2) r = outer_reward(state, a_v2) - assert 0.0 <= r <= 1.0, f"V2-rejected: r={r!r}" + assert -1.0 <= r <= 1.0, f"V2-rejected: r={r!r}" # Legal-violation: strict severity before escalate-national on hard. state2 = load_task("outbreak_hard", episode_seed=0) @@ -89,4 +91,4 @@ def test_outer_reward_in_range_with_rejected_actions() -> None: state2 = apply_tick(state2, a_legal) assert state2.recent_action_log[-1].accepted is False r2 = outer_reward(state2, a_legal) - assert 0.0 <= r2 <= 1.0, f"legal-violation: r={r2!r}" + assert -1.0 <= r2 <= 1.0, f"legal-violation: r={r2!r}" diff --git a/tests/test_outer_reward_non_constancy.py b/tests/test_outer_reward_non_constancy.py index bdf2b721da23923b002f720c78f7b50ea0dbfcd1..fac3bc7aef80716c015042985ce633345ed70230 100644 --- a/tests/test_outer_reward_non_constancy.py +++ b/tests/test_outer_reward_non_constancy.py @@ -15,6 +15,11 @@ from CrisisWorldCortex.models import ( RestrictMovement, ) from CrisisWorldCortex.server.graders import outer_reward +from CrisisWorldCortex.server.graders.outer_reward import ( + R_POLICY_NOOP_ACCEPTED, + R_POLICY_REJECTED, + W_POLICY, +) from CrisisWorldCortex.server.simulator import apply_tick, load_task @@ -87,8 +92,12 @@ def test_reward_differs_for_accepted_vs_rejected_action() -> None: Compare two episodes from the same starting state: one issues NoOp (accepted), the other issues PublicCommunication (V2-rejected). The - SEIR step runs identically; only ``r_policy`` differs. With weight - 0.12, the gap should be exactly 0.12 (modulo float rounding). + SEIR step runs identically; only ``r_policy`` differs. After the + Workstream-B Phase-1 four-state contract, NoOp(accepted) → + R_POLICY_NOOP_ACCEPTED (0.0) and PublicCommunication(honesty=0.9, + rejected as legal-violation) → R_POLICY_REJECTED (-0.5). The exact + gap is therefore ``(R_POLICY_NOOP_ACCEPTED - R_POLICY_REJECTED) * + W_POLICY`` (modulo float rounding). """ s_a = load_task("outbreak_easy", episode_seed=42) s_b = load_task("outbreak_easy", episode_seed=42) @@ -109,7 +118,7 @@ def test_reward_differs_for_accepted_vs_rejected_action() -> None: assert r_ok > r_bad, ( f"accepted action should score higher than rejected: ok={r_ok!r} bad={r_bad!r}" ) - # Exact gap = 0.12 because the only differing component is r_policy. - assert abs((r_ok - r_bad) - 0.12) < 1e-9, ( - f"r_policy gap mismatch: ok-bad={r_ok - r_bad!r}, expected 0.12" + expected_gap = (R_POLICY_NOOP_ACCEPTED - R_POLICY_REJECTED) * W_POLICY + assert abs((r_ok - r_bad) - expected_gap) < 1e-9, ( + f"r_policy gap mismatch: ok-bad={r_ok - r_bad!r}, expected {expected_gap!r}" ) diff --git a/tests/test_reward_signal_quality.py b/tests/test_reward_signal_quality.py new file mode 100644 index 0000000000000000000000000000000000000000..7b6bbc0b6003c28e3e74a2dd86e38d2feb08eb2b --- /dev/null +++ b/tests/test_reward_signal_quality.py @@ -0,0 +1,163 @@ +"""Reward signal-quality gates (Workstream B Phase 1). + +Locks the post-fix reward function as a permanent regression gate. Any +future reward change that breaks these tests fails loudly. Targets are +the relaxed Phase-1-crunch values: + + - all_no_op mean per-tick < 0.40 on outbreak_easy + - all_rejected mean per-tick < 0.40 on outbreak_easy + - active_strategic mean per-tick > 0.50 on outbreak_easy + - parse_failure on tick 1 sets done=True at tick 1 + - signal_separation = mean(active) - mean(no_op) >= 0.20 + +Per design §15 / §19 contract restoration: + - r_policy ∈ {-1.0 (parse_failure), -0.5 (rejected), 0.0 (accepted no-op), + +1.0 (accepted real action)} + - parse_failure_marker (PublicCommunication with honesty=0.0) + terminates the episode as state.terminal == "failure". +""" + +from __future__ import annotations + +from CrisisWorldCortex.models import ( + CrisisworldcortexAction, + DeployResource, + NoOp, + PublicCommunication, + RestrictMovement, +) +from CrisisWorldCortex.server.CrisisWorldCortex_environment import ( + CrisisworldcortexEnvironment, +) + +NO_OP_THRESHOLD = 0.40 +REJECTED_THRESHOLD = 0.40 +ACTIVE_THRESHOLD = 0.50 +SEPARATION_THRESHOLD = 0.20 + +EPISODE_TICKS = 12 +TASK = "outbreak_easy" +SEED = 0 + + +def _mean_per_tick_reward(rewards: list[float]) -> float: + """Mean of per-tick obs.reward across an episode.""" + if not rewards: + return 0.0 + return sum(rewards) / len(rewards) + + +def _run_action_sequence(actions: list) -> tuple[list[float], list[bool]]: + """Run one episode with a fixed action sequence. + + Returns (rewards_per_tick, done_per_tick). Runs until ``max_ticks`` or + ``done == True``; whichever comes first. + """ + env = CrisisworldcortexEnvironment() + env.reset(task_name=TASK, seed=SEED, max_ticks=EPISODE_TICKS) + rewards: list[float] = [] + dones: list[bool] = [] + for action_payload in actions: + obs = env.step(CrisisworldcortexAction(action=action_payload)) + rewards.append(obs.reward if obs.reward is not None else 0.0) + dones.append(bool(obs.done)) + if obs.done: + break + return rewards, dones + + +def test_all_no_op_episode_scores_below_threshold() -> None: + """12 ticks of NoOp on outbreak_easy → mean per-tick reward < 0.40. + + NoOp is a *legal* action, so r_policy = 0.0 (per Phase A M4). The + other components stay near 1.0 on outbreak_easy because the env is + gentle, but the reweighted W_POLICY (0.35) on a 0.0 r_policy makes + no_op a structurally low-scoring trajectory. + """ + actions = [NoOp() for _ in range(EPISODE_TICKS)] + rewards, dones = _run_action_sequence(actions) + mean_reward = _mean_per_tick_reward(rewards) + assert mean_reward < NO_OP_THRESHOLD, ( + f"all_no_op mean reward {mean_reward:.3f} >= threshold {NO_OP_THRESHOLD}; " + f"per-tick rewards={rewards!r}" + ) + + +def test_all_rejected_episode_scores_below_threshold() -> None: + """12 ticks of well-formed-illegal RestrictMovement(R1, strict) → < 0.40. + + On outbreak_easy without prior Escalate(national), strict severity is + legal-violation rejected. r_policy = -0.5 every tick. The reward + drops below the no_op floor because of the explicit -0.5 penalty. + """ + actions = [RestrictMovement(region="R1", severity="strict") for _ in range(EPISODE_TICKS)] + rewards, dones = _run_action_sequence(actions) + mean_reward = _mean_per_tick_reward(rewards) + assert mean_reward < REJECTED_THRESHOLD, ( + f"all_rejected mean reward {mean_reward:.3f} >= threshold {REJECTED_THRESHOLD}; " + f"per-tick rewards={rewards!r}" + ) + + +def test_active_strategic_episode_scores_above_threshold() -> None: + """12 ticks of valid DeployResource(R1, test_kits, 100) → mean > 0.50. + + Real accepted action gets r_policy = 1.0 every tick. With infection + suppression keeping the steepened r_infect / r_hosp components high, + the weighted score should clear 0.50 comfortably. + """ + actions = [ + DeployResource(region="R1", resource_type="test_kits", quantity=100) + for _ in range(EPISODE_TICKS) + ] + rewards, dones = _run_action_sequence(actions) + mean_reward = _mean_per_tick_reward(rewards) + assert mean_reward > ACTIVE_THRESHOLD, ( + f"active_strategic mean reward {mean_reward:.3f} <= threshold {ACTIVE_THRESHOLD}; " + f"per-tick rewards={rewards!r}" + ) + + +def test_parse_failure_terminates_episode() -> None: + """Single parse-failure marker on tick 1 → done=True, reward < 0. + + The synthetic parse_failure_marker (PublicCommunication with + honesty=0.0) is the magic-string discriminator for parse-failure + rejection per Phase A M3-B. The env must set state.terminal = + "failure" on this rejection, propagating to obs.done = True. + """ + parse_failure = PublicCommunication( + audience="general", + message_class="informational", + honesty=0.0, + ) + rewards, dones = _run_action_sequence([parse_failure]) + assert len(dones) == 1, f"expected episode to terminate at tick 1, got {len(dones)} ticks" + assert dones[0] is True, f"parse_failure tick 1 did not set done=True; dones={dones!r}" + # r_policy = -1.0 dominates → final reward should be negative. + assert rewards[0] < 0.0, ( + f"parse_failure reward {rewards[0]:.3f} should be negative (r_policy=-1.0 contract)" + ) + + +def test_signal_separation() -> None: + """Active-vs-no_op gap >= 0.20 — locks the trainable gradient. + + This is the mathematical floor on what GRPO (or any policy-gradient + method) can extract from the reward signal. If active and no_op score + similarly, training has nothing to optimise toward. + """ + no_op_actions = [NoOp() for _ in range(EPISODE_TICKS)] + active_actions = [ + DeployResource(region="R1", resource_type="test_kits", quantity=100) + for _ in range(EPISODE_TICKS) + ] + no_op_rewards, _ = _run_action_sequence(no_op_actions) + active_rewards, _ = _run_action_sequence(active_actions) + no_op_mean = _mean_per_tick_reward(no_op_rewards) + active_mean = _mean_per_tick_reward(active_rewards) + separation = active_mean - no_op_mean + assert separation >= SEPARATION_THRESHOLD, ( + f"signal_separation {separation:.3f} < threshold {SEPARATION_THRESHOLD}; " + f"no_op_mean={no_op_mean:.3f}, active_mean={active_mean:.3f}" + ) diff --git a/tests/test_schemas_roundtrip.py b/tests/test_schemas_roundtrip.py index e6e14ae0fda26083575536f94be76042d02f3ee0..577e579d579ec89cfc1f4be0811b422521bea83e 100644 --- a/tests/test_schemas_roundtrip.py +++ b/tests/test_schemas_roundtrip.py @@ -8,6 +8,7 @@ types in real flows. from cortex.schemas import ( BeliefState, + BrainLensedObservation, BrainRecommendation, CandidatePlan, CouncilDecision, @@ -18,6 +19,7 @@ from cortex.schemas import ( PerceptionReport, RegionBeliefEstimate, RoutingAction, + SubagentInput, ) from CrisisWorldCortex.models import ( CrisisworldcortexAction, @@ -210,3 +212,62 @@ def test_perception_and_critic_roundtrips() -> None: severity=0.6, ) assert CriticReport.model_validate_json(cr.model_dump_json()) == cr + + +# T7 (Session 9) -- SubagentInput round-trip +def test_subagent_input_roundtrip() -> None: + si = SubagentInput( + brain="epidemiology", + role="world_modeler", + tick=3, + round=1, + perception=PerceptionReport( + brain="epidemiology", + salient_signals=["R1 cases rising"], + anomalies=[], + confidence=0.7, + evidence=[EvidenceCitation(source="telemetry", ref="R1.cases", excerpt="rising")], + ), + prior_belief=None, + prior_plans=[], + target_plan_id=None, + last_reward=0.5, + recent_action_log_excerpt=[], + ) + restored = SubagentInput.model_validate_json(si.model_dump_json()) + assert restored == si + assert restored.role == "world_modeler" + assert restored.brain == "epidemiology" + + +# T6 (Session 10) -- BrainLensedObservation round-trip +def test_brain_lensed_observation_roundtrip() -> None: + obs = CrisisworldcortexObservation( + regions=[ + RegionTelemetry( + region="R1", reported_cases_d_ago=5, hospital_load=0.3, compliance_proxy=0.85 + ), + ], + resources=ResourceInventory(test_kits=100, hospital_beds_free=50), + active_restrictions=[Restriction(region="R1", severity="moderate", ticks_remaining=3)], + legal_constraints=[], + tick=3, + ticks_remaining=9, + cognition_budget_remaining=5200, + recent_action_log=[], + ) + blo = BrainLensedObservation( + brain="epidemiology", + raw_obs=obs, + salient_field_ids=["regions[*].hospital_load", "regions[*].reported_cases_d_ago"], + derived_features={ + "epi_pressure": 0.6, + "worst_region_infection": 0.005, + "transmission_rate_trend": 0.0, + }, + last_reward=0.5, + ) + restored = BrainLensedObservation.model_validate_json(blo.model_dump_json()) + assert restored == blo + assert restored.brain == "epidemiology" + assert restored.raw_obs == obs diff --git a/tests/test_training_eval_metrics.py b/tests/test_training_eval_metrics.py new file mode 100644 index 0000000000000000000000000000000000000000..cf2c9677ee8d2de558b77bbf95d2f5af33a3eff4 --- /dev/null +++ b/tests/test_training_eval_metrics.py @@ -0,0 +1,54 @@ +"""Tests for ``training.eval_metrics``. + +``collapse_rate`` is fully exercised against synthetic B1-style +trajectories; the three Cortex-dependent stubs (``dissent_value``, +``consensus_calibration``, ``novelty_yield``) get smoke checks that +assert their Phase-2 placeholder return value of 0.0. +""" + +from __future__ import annotations + +from training.eval_metrics import ( + collapse_rate, + consensus_calibration, + dissent_value, + novelty_yield, +) + + +def _trajectory(action_kinds: list[str]) -> list[dict]: + """Build a synthetic B1-style trajectory from a list of action kinds.""" + return [{"action": {"kind": k}} for k in action_kinds] + + +def test_collapse_rate_zero_when_actions_diverse() -> None: + """Diverse trajectory (no modal action >= 80%) → 0.0.""" + trajs = [_trajectory(["no_op", "deploy_resource", "restrict_movement", "escalate"])] + assert collapse_rate(trajs) == 0.0 + + +def test_collapse_rate_one_when_all_actions_identical() -> None: + """All-NoOp 12-tick episode → fully collapsed.""" + trajs = [_trajectory(["no_op"] * 12)] + assert collapse_rate(trajs) == 1.0 + + +def test_collapse_rate_partial_for_mixed_episodes() -> None: + """Two episodes, one collapsed and one diverse → 0.5.""" + collapsed = _trajectory(["no_op"] * 10) + diverse = _trajectory(["no_op", "deploy_resource", "restrict_movement", "no_op"]) + assert collapse_rate([collapsed, diverse]) == 0.5 + + +def test_collapse_rate_short_episodes_excluded() -> None: + """Episodes shorter than COLLAPSE_MIN_STEPS don't qualify.""" + trajs = [_trajectory(["no_op"] * 2)] # below 3-step minimum + assert collapse_rate(trajs) == 0.0 + + +def test_cortex_dependent_metrics_return_zero_in_phase_2() -> None: + """dissent/consensus/novelty stubs return 0.0 until Cortex Session 13.""" + trajs = [_trajectory(["no_op", "deploy_resource", "no_op"])] + assert dissent_value(trajs) == 0.0 + assert consensus_calibration(trajs) == 0.0 + assert novelty_yield(trajs) == 0.0 diff --git a/tests/test_training_reward_shaping.py b/tests/test_training_reward_shaping.py new file mode 100644 index 0000000000000000000000000000000000000000..4ea08bd0565ee6a846d6466befb46a33d43738a5 --- /dev/null +++ b/tests/test_training_reward_shaping.py @@ -0,0 +1,57 @@ +"""Tests for ``training.reward_shaping``. + +Token-budget penalty composition + episode-return terminal bonus. +""" + +from __future__ import annotations + +import pytest + +from training.reward_shaping import ( + DEFAULT_LAMBDA_BUDGET, + DEFAULT_TICK_BUDGET, + compose_episode_return, + shape_reward, +) + + +def test_shape_reward_zero_tokens_returns_outer_unchanged() -> None: + """tokens_used=0 → r_total == r_outer (no penalty).""" + assert shape_reward(0.7, tokens_used=0) == 0.7 + assert shape_reward(-0.3, tokens_used=0) == -0.3 + + +def test_shape_reward_full_budget_applies_full_penalty() -> None: + """tokens_used == tick_budget → penalty == lambda_budget.""" + r_outer = 1.0 + out = shape_reward(r_outer, tokens_used=DEFAULT_TICK_BUDGET) + expected = r_outer - DEFAULT_LAMBDA_BUDGET * 1.0 + assert abs(out - expected) < 1e-9 + + +def test_shape_reward_partial_budget_scales_linearly() -> None: + """Half-budget consumption → half the lambda penalty.""" + r_outer = 0.5 + half = DEFAULT_TICK_BUDGET // 2 + out = shape_reward(r_outer, tokens_used=half, lambda_budget=0.4) + expected = 0.5 - 0.4 * 0.5 + assert abs(out - expected) < 1e-9 + + +def test_shape_reward_invalid_tick_budget_raises() -> None: + """tick_budget <= 0 → ValueError (guard against div-by-zero).""" + with pytest.raises(ValueError, match="positive"): + shape_reward(0.5, tokens_used=10, tick_budget=0) + + +def test_compose_episode_return_terminal_bonus_success() -> None: + """Success terminal adds +0.20 to summed rewards.""" + assert abs(compose_episode_return([0.1, 0.2, 0.3], "success") - 0.80) < 1e-9 + + +def test_compose_episode_return_terminal_bonus_failure_and_neutral() -> None: + """Failure → -0.20; timeout/none → no bonus.""" + base = [0.1, 0.1, 0.1] + assert abs(compose_episode_return(base, "failure") - 0.10) < 1e-9 + assert abs(compose_episode_return(base, "timeout") - 0.30) < 1e-9 + assert abs(compose_episode_return(base, "none") - 0.30) < 1e-9 diff --git a/tests/test_training_rollout_buffer.py b/tests/test_training_rollout_buffer.py new file mode 100644 index 0000000000000000000000000000000000000000..513ae700a7e4b84ac26753b6aa89227e6e1e940a --- /dev/null +++ b/tests/test_training_rollout_buffer.py @@ -0,0 +1,78 @@ +"""Tests for ``training.rollout_buffer``. + +Round-trip add/get, group-sampling contract, clear-empties-buffer. +""" + +from __future__ import annotations + +import random + +import pytest + +from training.rollout_buffer import RolloutBuffer, TrajectoryStep + + +def _make_step(reward: float = 0.5, done: bool = False) -> TrajectoryStep: + return TrajectoryStep( + obs={"tick": 0, "regions": []}, + action={"kind": "no_op"}, + reward=reward, + log_prob=None, + done=done, + ) + + +def test_rollout_buffer_round_trip() -> None: + """add_step → get_episode round-trips without data loss.""" + buf = RolloutBuffer() + s1 = _make_step(reward=0.3) + s2 = _make_step(reward=0.7, done=True) + buf.add_step("ep-A", s1) + buf.add_step("ep-A", s2) + out = buf.get_episode("ep-A") + assert out == [s1, s2] + assert buf.get_episode("missing-id") == [] + + +def test_rollout_buffer_episode_return_sums_rewards() -> None: + """episode_return == sum of step rewards.""" + buf = RolloutBuffer() + buf.add_step("ep", _make_step(reward=0.1)) + buf.add_step("ep", _make_step(reward=-0.2)) + buf.add_step("ep", _make_step(reward=0.5)) + assert abs(buf.episode_return("ep") - 0.4) < 1e-9 + + +def test_rollout_buffer_sample_group_is_deterministic_with_seed() -> None: + """Same seed → same sample; different seed → likely different sample. + + Locks GRPO group-sampling determinism for trainer reproducibility. + """ + buf = RolloutBuffer() + for i in range(8): + buf.add_step(f"ep-{i}", _make_step()) + rng_a = random.Random(42) + rng_b = random.Random(42) + sample_a = sorted(buf.sample_group(4, rng=rng_a)) + sample_b = sorted(buf.sample_group(4, rng=rng_b)) + assert sample_a == sample_b + assert len(set(sample_a)) == 4 # no duplicates + + +def test_rollout_buffer_sample_group_oversize_raises() -> None: + """Requesting more episodes than buffer holds → ValueError.""" + buf = RolloutBuffer() + buf.add_step("ep-0", _make_step()) + with pytest.raises(ValueError, match="exceeds buffer size"): + buf.sample_group(5) + + +def test_rollout_buffer_clear_empties_state() -> None: + """clear() empties the buffer; len → 0.""" + buf = RolloutBuffer() + for i in range(3): + buf.add_step(f"ep-{i}", _make_step()) + assert len(buf) == 3 + buf.clear() + assert len(buf) == 0 + assert buf.episode_ids() == [] diff --git a/training/CLAUDE.md b/training/CLAUDE.md index 6880a5c7677becae7851d7ea70862f7539cffc20..3a4e84711fc0673c20b720a42224889b85c8f40a 100644 --- a/training/CLAUDE.md +++ b/training/CLAUDE.md @@ -1,64 +1,64 @@ -# training/CLAUDE.md - -GRPO training for the routing policy. Two Colab notebooks required for hackathon compliance. - -## Belongs here - -- `train_router.py` — manual PyTorch GRPO loop for the MLP router (Option B primary). -- `rollout_buffer.py` — trajectory collection + router-step serialization. -- `reward_shaping.py` — composes the 4-term training reward from `server.graders` outputs. -- `train_router_colab.ipynb` — Colab-runnable notebook for the MLP router. **Required.** -- `train_flat_agent_trl.ipynb` — TRL `GRPOTrainer` + Unsloth LoRA on a flat agent against `outbreak_easy`. **Required** for finale Unsloth/TRL compliance (design §E.3). -- `configs/grpo_config.yaml` — GRPO hyperparameters. -- `configs/tasks.yaml` — task-curriculum config (easy → medium → hard). - -## Does not belong here - -Baseline agents (→ `baselines/`). Plotting / reward curves (→ `demo/`, `scripts/`). SEIR dynamics (→ `server/simulator/`). - -## Allowed imports - -- `models`, `client`. -- `cortex.routing_policy`, `cortex.council`, `cortex.schemas`, `cortex.metacognition`. -- `server.graders` — **reward-name constants only** (e.g. the `training_reward` dict keys). Do not import `server.simulator`. Do not instantiate the env in-process. -- Torch, TRL, Unsloth (compliance notebook only). - -## Forbidden imports - -- `server.simulator/*` — training hits the env over HTTP like production. -- `baselines/*`, `demo/*`. - -## Binding contracts - -- **Training-data rows = router steps**, not ticks or rounds. One row per `RoutingAction` emission. -- **Training reward = exactly the 4 terms returned by `server.graders.training_reward`** (see `server/CLAUDE.md` for the dict schema). Never mix in eval-only metrics. -- Training episode length = 10–12 ticks. Eval episode length = 20 ticks (only if training is stable; otherwise eval also runs at 12). -- Temperature > 0 on LLM subagents during rollouts (exploration); temperature = 0 during eval (reproducibility). -- Pin the OpenEnv version in `pyproject.toml` before training runs — finale requires "latest release" at submission. - -## Colab notebook contracts - -- `train_router_colab.ipynb`: imports CrisisWorld as a local Python module (no Docker in Colab). Runs end-to-end on a fresh Colab T4. -- `train_flat_agent_trl.ipynb`: uses `trl.GRPOTrainer(environment_factory=CrisisworldcortexEnv, ...)` + `unsloth.FastLanguageModel` LoRA wrapper. Runs end-to-end on a fresh Colab T4. **This notebook's absence disqualifies the submission.** - -## Public APIs (owned here) - -- `train_router.main(config_path: str) -> None` -- `RolloutBuffer.add(router_step: RouterStep) -> None` -- `RolloutBuffer.sample(batch_size: int) -> list[RouterStep]` -- `shape_reward(trajectory: Trajectory) -> float` — weighted combination of the 4 training terms. - -## Testing requirements - -- `shape_reward` returns a scalar in `[0.0, 1.0]`. -- `RolloutBuffer` round-trips synthetic router steps without data loss. -- `train_router.py` runs 1 episode end-to-end against a mocked env in under 5 seconds (CI smoke). -- Both Colab notebooks execute to completion for ≥ a few hundred training steps on Colab T4 pre-onsite. - -## Common failure modes - -- Logging ticks as training rows — collapses router's action granularity; GRPO credit assignment breaks. -- Mixing eval metrics into the training reward — inflates the headline curve for reasons the paper can't defend. -- Widening training episodes past 12 ticks — rollouts stop fitting in the GRPO update window; wall-clock explodes. -- Training-reward dict keys drifting from `server.graders.training_reward` — shape-only tests miss this; trainer silently optimizes the wrong signal. -- Missing the TRL compliance notebook at submission — automatic finale failure. +# training/CLAUDE.md + +GRPO training for the routing policy. Two Colab notebooks required for hackathon compliance. + +## Belongs here + +- `train_router.py` — manual PyTorch GRPO loop for the MLP router (Option B primary). +- `rollout_buffer.py` — trajectory collection + router-step serialization. +- `reward_shaping.py` — composes the 4-term training reward from `server.graders` outputs. +- `train_router_colab.ipynb` — Colab-runnable notebook for the MLP router. **Required.** +- `train_flat_agent_trl.ipynb` — TRL `GRPOTrainer` + Unsloth LoRA on a flat agent against `outbreak_easy`. **Required** for finale Unsloth/TRL compliance (design §E.3). +- `configs/grpo_config.yaml` — GRPO hyperparameters. +- `configs/tasks.yaml` — task-curriculum config (easy → medium → hard). + +## Does not belong here + +Baseline agents (→ `baselines/`). Plotting / reward curves (→ `demo/`, `scripts/`). SEIR dynamics (→ `server/simulator/`). + +## Allowed imports + +- `models`, `client`. +- `cortex.routing_policy`, `cortex.council`, `cortex.schemas`, `cortex.metacognition`. +- `server.graders` — **reward-name constants only** (e.g. the `training_reward` dict keys). Do not import `server.simulator`. Do not instantiate the env in-process. +- Torch, TRL, Unsloth (compliance notebook only). + +## Forbidden imports + +- `server.simulator/*` — training hits the env over HTTP like production. +- `baselines/*`, `demo/*`. + +## Binding contracts + +- **Training-data rows = router steps**, not ticks or rounds. One row per `RoutingAction` emission. +- **Training reward = exactly the 4 terms returned by `server.graders.training_reward`** (see `server/CLAUDE.md` for the dict schema). Never mix in eval-only metrics. +- Training episode length = 10–12 ticks. Eval episode length = 20 ticks (only if training is stable; otherwise eval also runs at 12). +- Temperature > 0 on LLM subagents during rollouts (exploration); temperature = 0 during eval (reproducibility). +- Pin the OpenEnv version in `pyproject.toml` before training runs — finale requires "latest release" at submission. + +## Colab notebook contracts + +- `train_router_colab.ipynb`: imports CrisisWorld as a local Python module (no Docker in Colab). Runs end-to-end on a fresh Colab T4. +- `train_flat_agent_trl.ipynb`: uses `trl.GRPOTrainer(environment_factory=CrisisworldcortexEnv, ...)` + `unsloth.FastLanguageModel` LoRA wrapper. Runs end-to-end on a fresh Colab T4. **This notebook's absence disqualifies the submission.** + +## Public APIs (owned here) + +- `train_router.main(config_path: str) -> None` +- `RolloutBuffer.add(router_step: RouterStep) -> None` +- `RolloutBuffer.sample(batch_size: int) -> list[RouterStep]` +- `shape_reward(trajectory: Trajectory) -> float` — weighted combination of the 4 training terms. + +## Testing requirements + +- `shape_reward` returns a scalar in `[0.0, 1.0]`. +- `RolloutBuffer` round-trips synthetic router steps without data loss. +- `train_router.py` runs 1 episode end-to-end against a mocked env in under 5 seconds (CI smoke). +- Both Colab notebooks execute to completion for ≥ a few hundred training steps on Colab T4 pre-onsite. + +## Common failure modes + +- Logging ticks as training rows — collapses router's action granularity; GRPO credit assignment breaks. +- Mixing eval metrics into the training reward — inflates the headline curve for reasons the paper can't defend. +- Widening training episodes past 12 ticks — rollouts stop fitting in the GRPO update window; wall-clock explodes. +- Training-reward dict keys drifting from `server.graders.training_reward` — shape-only tests miss this; trainer silently optimizes the wrong signal. +- Missing the TRL compliance notebook at submission — automatic finale failure. diff --git a/training/__init__.py b/training/__init__.py index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..408c1e33c6c2bc4bace96762184c6193f2be7b92 100644 --- a/training/__init__.py +++ b/training/__init__.py @@ -0,0 +1,32 @@ +"""Training package: rollout buffer, reward shaping, eval metrics. + +Phase-2 scaffold (Workstream B). The actual GRPO trainer +(``training/train_router.py``) lands in Session 15 of Workstream A. +""" + +from .eval_metrics import ( + collapse_rate, + consensus_calibration, + dissent_value, + novelty_yield, +) +from .reward_shaping import ( + DEFAULT_LAMBDA_BUDGET, + DEFAULT_TICK_BUDGET, + compose_episode_return, + shape_reward, +) +from .rollout_buffer import RolloutBuffer, TrajectoryStep + +__all__ = [ + "DEFAULT_LAMBDA_BUDGET", + "DEFAULT_TICK_BUDGET", + "RolloutBuffer", + "TrajectoryStep", + "collapse_rate", + "compose_episode_return", + "consensus_calibration", + "dissent_value", + "novelty_yield", + "shape_reward", +] diff --git a/training/eval_metrics.py b/training/eval_metrics.py new file mode 100644 index 0000000000000000000000000000000000000000..ba70accdae0b21dbf83c8b95c0d7694aef153aa4 --- /dev/null +++ b/training/eval_metrics.py @@ -0,0 +1,119 @@ +"""Eval-only metrics for trajectories (design §20). + +Four signals, NOT optimised by the trainer (per ``server/CLAUDE.md`` +binding contract: training_reward and eval_metrics live in separate +trajectory-log columns and must never combine): + + - ``collapse_rate(trajectories)`` — fraction of episodes where the + agent collapsed to the same action repeatedly (proxy for hivemind + failure / over-exploitation). + - ``dissent_value(trajectories)`` — Cortex-only: how often a preserved + minority recommendation later proved correct (next-tick re-check). + Returns ``0.0`` until Cortex Session 13 lands real preserved-dissent + metadata; trajectory shape allows the metric to round-trip without + breaking. + - ``consensus_calibration(trajectories)`` — Cortex-only: correlation + between brain-reported confidence and realised reward. Returns + ``0.0`` until Cortex lands; placeholder. + - ``novelty_yield(trajectories)`` — Cortex-only: fraction of round-2 + deliberations that produced a different action than round-1. + Returns ``0.0`` until Cortex lands; placeholder. + +Phase-2 scaffold (Workstream B). ``collapse_rate`` is fully implemented +against B1/B2-style trajectories (action_history with ``"action"`` dicts); +the three Cortex-dependent metrics are stubbed with TODOs marking what +Cortex Session 13 will need to populate. + +Allowed under ``training/CLAUDE.md``: ``models``, stdlib only. +""" + +from __future__ import annotations + +from typing import Iterable + +# Threshold: an episode is "collapsed" if at least this fraction of its +# steps used the modal action. 0.8 captures "policy went lazy" without +# false-flagging short episodes that legitimately repeat one action. +COLLAPSE_FRACTION_THRESHOLD = 0.8 +COLLAPSE_MIN_STEPS = 3 # ignore episodes too short to be diagnostic + + +def _action_kind(step: dict) -> str: + """Extract action-kind discriminator from a trajectory step dict.""" + action = step.get("action", {}) + if isinstance(action, dict): + return str(action.get("kind", "unknown")) + return getattr(action, "kind", "unknown") + + +def collapse_rate(trajectories: Iterable[list[dict]]) -> float: + """Fraction of episodes that collapsed to a single modal action. + + Modal action share >= COLLAPSE_FRACTION_THRESHOLD in episodes with + >= COLLAPSE_MIN_STEPS counts as a "collapsed" episode. Returns the + fraction of input episodes that collapsed; 0.0 if no episodes + qualify (all too short). + """ + qualifying = 0 + collapsed = 0 + for traj in trajectories: + if len(traj) < COLLAPSE_MIN_STEPS: + continue + qualifying += 1 + kinds = [_action_kind(step) for step in traj] + modal_count = max(kinds.count(k) for k in set(kinds)) + if modal_count / len(kinds) >= COLLAPSE_FRACTION_THRESHOLD: + collapsed += 1 + if qualifying == 0: + return 0.0 + return collapsed / qualifying + + +def dissent_value(trajectories: Iterable[list[dict]]) -> float: + """Cortex-only: preserved-dissent realised-correctness rate. + + Returns 0.0 in Phase-2; Cortex Session 13 wires the real metric. + + TODO(cortex-session-13): when ``RouterStep.preserved_dissent`` lands, + replace this stub with: for each tick where a minority rec was + preserved, check whether the next tick's chosen action matches the + preserved minority. dissent_value = matches / preservations. + """ + # Touch the iterator to surface accidental misuse (e.g. passing a + # generator that the caller still expects to be unconsumed). + _ = list(trajectories) + return 0.0 + + +def consensus_calibration(trajectories: Iterable[list[dict]]) -> float: + """Cortex-only: correlation between confidence and realised reward. + + Returns 0.0 in Phase-2; Cortex Session 13 wires the real metric. + + TODO(cortex-session-13): when ``BrainRecommendation.top_confidence`` + is in trajectory rows, compute Pearson correlation between mean + brain confidence and per-tick reward. Returns scalar in [-1, 1]. + """ + _ = list(trajectories) + return 0.0 + + +def novelty_yield(trajectories: Iterable[list[dict]]) -> float: + """Cortex-only: round-2 action-change rate vs round-1. + + Returns 0.0 in Phase-2; Cortex Session 13 wires the real metric. + + TODO(cortex-session-13): when ``RouterStep.round`` is populated, + compute fraction of ticks where round-2 emit_outer_action differs + from the round-1 candidate top_action. + """ + _ = list(trajectories) + return 0.0 + + +__all__ = [ + "collapse_rate", + "consensus_calibration", + "dissent_value", + "novelty_yield", +] diff --git a/training/reward_shaping.py b/training/reward_shaping.py new file mode 100644 index 0000000000000000000000000000000000000000..3ddcb4836973a1bb1bb79b93efd20151819ef27e --- /dev/null +++ b/training/reward_shaping.py @@ -0,0 +1,96 @@ +"""Training reward composition (design §17 + Phase-1 outer-reward range). + +Composes the env-side ``outer_reward`` (post-Phase-1 range ``[-1.0, 1.0]``) +with a token-budget penalty into the per-tick training reward used by +GRPO: + + r_total = r_outer - lambda_budget * (tokens_used / TICK_BUDGET) + +The terminal bonus (±0.20 per design §14.3) is added once per episode by +``compose_episode_return``, NOT folded into per-tick ``r_total``. + +Phase-2 scaffold (Workstream B). Trainer is responsible for clamping if +it needs a bounded range; this module returns the raw composition. + +Allowed under ``training/CLAUDE.md``: ``models``, stdlib, and +``server.graders`` reward-name constants. No ``server.simulator``, no +``cortex/*``, no ``baselines/*``. +""" + +from __future__ import annotations + +from CrisisWorldCortex.server.graders.outer_reward import ( + TERMINAL_BONUS_FAILURE, + TERMINAL_BONUS_SUCCESS, +) + +DEFAULT_TICK_BUDGET = 6000 # design §11.2; matches B2 default and Phase-A §5 +DEFAULT_LAMBDA_BUDGET = 0.5 # design §17 default; tunable per training run + + +def shape_reward( + outer_reward: float, + tokens_used: int, + *, + tick_budget: int = DEFAULT_TICK_BUDGET, + lambda_budget: float = DEFAULT_LAMBDA_BUDGET, +) -> float: + """Compose per-tick training reward. + + ``r_total = r_outer - lambda_budget * (tokens_used / tick_budget)`` + + The token-budget penalty drives the policy toward shorter rollouts; + ``lambda_budget`` controls how aggressive that pressure is. Returns + a raw scalar (no clamp) — caller decides how to bound for trainer + consumption. + + Args: + outer_reward: ``server.graders.outer_reward`` output, ``[-1, 1]``. + tokens_used: LLM tokens spent on this tick (harness-counted via + ``cortex.llm_client.LLMClient.tokens_used_for(...)``). + tick_budget: Per-tick token cap (default ``6000``). + lambda_budget: Penalty coefficient (default ``0.5``). + + Returns: + Per-tick training reward (``r_total``), unbounded. + """ + if tick_budget <= 0: + raise ValueError(f"tick_budget must be positive; got {tick_budget!r}") + budget_fraction = tokens_used / tick_budget + return outer_reward - lambda_budget * budget_fraction + + +def compose_episode_return( + per_tick_rewards: list[float], + terminal_kind: str, +) -> float: + """Sum per-tick rewards + terminal bonus per design §14.3. + + ``episode_return = sum(per_tick_rewards) + terminal_bonus(terminal_kind)`` + + where ``terminal_bonus`` is ``+0.20`` on ``"success"``, ``-0.20`` on + ``"failure"``, and ``0.0`` on ``"timeout"`` / ``"none"``. + + Per-tick rewards must already be shaped via ``shape_reward(...)`` if + the trainer wants the token-budget penalty included. This function + is the final aggregation step. + + Args: + per_tick_rewards: List of per-tick training rewards. + terminal_kind: One of ``"success"``, ``"failure"``, ``"timeout"``, + ``"none"`` (matches ``state.terminal``). + """ + base = sum(per_tick_rewards) + if terminal_kind == "success": + return base + TERMINAL_BONUS_SUCCESS + if terminal_kind == "failure": + return base + TERMINAL_BONUS_FAILURE + return base + + +__all__ = [ + "DEFAULT_LAMBDA_BUDGET", + "DEFAULT_TICK_BUDGET", + "compose_episode_return", + "shape_reward", +] diff --git a/training/rollout_buffer.py b/training/rollout_buffer.py new file mode 100644 index 0000000000000000000000000000000000000000..59e5969edba7de913dca4ae62e86e57e11ebfa71 --- /dev/null +++ b/training/rollout_buffer.py @@ -0,0 +1,95 @@ +"""Rollout buffer for GRPO training. + +Stores per-episode ``TrajectoryStep`` tuples, supports group sampling for +GRPO's relative-advantage computation, and exposes a clear/round-trip +contract for unit tests. Generic shape: works for B1 / B2 / Cortex +trajectories without coupling to any specific agent implementation. + +Phase-2 scaffold (Workstream B). The buffer is intentionally minimal — +no batching, no tensor conversion, no on-disk persistence. Those land +in the actual GRPO trainer (``training/train_router.py``) when Session +15 implements it. + +Allowed under ``training/CLAUDE.md`` import rules: ``models`` and stdlib +only. No ``cortex/*``, no ``server/*``, no ``baselines/*``. +""" + +from __future__ import annotations + +import random +from dataclasses import dataclass, field +from typing import Optional + + +@dataclass(frozen=True) +class TrajectoryStep: + """One (obs, action, reward, log_prob, done) tuple from a rollout. + + ``obs`` and ``action`` are serialised to ``dict`` (via + ``BaseModel.model_dump()`` at the call site) so the buffer never + holds Pydantic objects directly — keeps GRPO trainer hot path free + of validation overhead. + + ``log_prob`` is the policy's log-probability of the chosen action + under the rollout-time temperature. ``None`` for non-stochastic + baselines (e.g. B1 with temperature=0). + """ + + obs: dict + action: dict + reward: float + log_prob: Optional[float] + done: bool + + +@dataclass +class RolloutBuffer: + """Per-episode rollout storage with GRPO group-sampling support. + + Episodes are keyed by an arbitrary ``episode_id`` string; the trainer + is responsible for choosing IDs (typically ``f"{task}:{seed}:{run}"``). + """ + + _episodes: dict[str, list[TrajectoryStep]] = field(default_factory=dict) + + def add_step(self, episode_id: str, step: TrajectoryStep) -> None: + """Append one step to the named episode (creates it if absent).""" + self._episodes.setdefault(episode_id, []).append(step) + + def get_episode(self, episode_id: str) -> list[TrajectoryStep]: + """Return the step list for ``episode_id`` (empty if unknown).""" + return self._episodes.get(episode_id, []) + + def episode_ids(self) -> list[str]: + """Return all episode IDs currently in the buffer.""" + return list(self._episodes.keys()) + + def episode_return(self, episode_id: str) -> float: + """Sum of rewards for the named episode.""" + return sum(s.reward for s in self.get_episode(episode_id)) + + def sample_group(self, group_size: int, rng: Optional[random.Random] = None) -> list[str]: + """Sample ``group_size`` episode IDs without replacement for GRPO. + + GRPO's relative-advantage step requires a group of trajectories + from the same prompt; the trainer typically calls this once per + update step. Returns episode IDs (not the full step lists) so + the trainer can decide how to slice them into tensors. + + Raises ``ValueError`` if ``group_size`` exceeds the buffer size. + """ + if group_size > len(self._episodes): + raise ValueError(f"group_size={group_size} exceeds buffer size={len(self._episodes)}") + rng = rng or random.Random() + return rng.sample(list(self._episodes.keys()), group_size) + + def clear(self) -> None: + """Drop all episodes. Called between GRPO update steps.""" + self._episodes.clear() + + def __len__(self) -> int: + """Number of episodes currently stored.""" + return len(self._episodes) + + +__all__ = ["RolloutBuffer", "TrajectoryStep"] diff --git a/unsloth_2048.ipynb b/unsloth_2048.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..c2afa4534aa5980bbd76a0cf0f3136352cbc9bb0 --- /dev/null +++ b/unsloth_2048.ipynb @@ -0,0 +1,7000 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "GtkTGCuh6QZy" + }, + "source": [ + "# Reinforcement Learning with OpenAI gpt-oss-20b: Teaching an LLM to Play 2048\n", + "\n", + "In this tutorial, we'll teach OpenAI's open-source model **gpt-oss 20b** to generate winning strategies for the classic 2048 puzzle game using **reinforcement learning (RL)**. By the end, you'll understand how to:\n", + "\n", + "- Connect LLMs to game environments using **OpenEnv**\n", + "- Design reward functions that guide model behavior\n", + "- Train models with **GRPO** (Group Relative Policy Optimization)\n", + "- Prevent \"reward hacking\" with code sandboxing\n", + "\n", + "**Requirements:** This notebook runs on a free Tesla T4 Google Colab instance." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "hzPgFeIkZn9q" + }, + "source": [ + "## What is 2048?\n", + "\n", + "**2048** is an fun single-player sliding puzzle game created by Gabriele Cirulli in 2014. The game is played on a 4×4 grid where numbered tiles slide in four directions (up, down, left, right). When two tiles with the same number collide, they merge into one tile with their sum. The goal is to create a tile with the value **2048**—though skilled players can continue beyond that!\n", + "\n", + "The game requires strategic thinking: random moves quickly lead to a gridlock, while optimal play involves keeping high-value tiles in corners and building systematically.\n", + "\n", + "\n", + "\n", + "## Our Goal\n", + "\n", + "We'll use reinforcement learning to train **OpenAI gpt-oss-20b** to generate Python functions that implement winning 2048 strategies. Rather than playing move-by-move, the model will learn to write *code* that plays the game—a form of \"code generation as policy.\"" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "31KIMLJLnHET" + }, + "source": [ + "## Installation\n", + "\n", + "We need two key libraries for this tutorial:\n", + "\n", + "1. **[OpenEnv](https://github.com/meta-pytorch/OpenEnv)** - A unified interface to reinforcement learning environments. Traditional RL setups require installing and configuring each environment separately (Gym, OpenSpiel, Atari, etc.). OpenEnv provides a consistent API across all of them. Best of all, OpenEnv environments are available on Hugging Face Spaces, so we can connect to them remotely without any local installation.\n", + "\n", + "2. **[Unsloth](https://github.com/unslothai/unsloth)** - An optimized training library that reduces VRAM usage by ~70% through memory-efficient LoRA and gradient checkpointing. This lets us run RL on a free Colab T4 GPU." + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": { + "id": "CGoDZwcunHEU" + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + " \u001b[31m×\u001b[0m No solution found when resolving dependencies:\n", + "\u001b[31m ╰─▶ \u001b[0mBecause triton==3.6.0 has no wheels with a matching platform tag (e.g.,\n", + "\u001b[31m \u001b[0m`\u001b[36mwin_amd64\u001b[39m`) and you require triton==3.6.0, we can conclude that your\n", + "\u001b[31m \u001b[0mrequirements are unsatisfiable.\n", + "\n", + "\u001b[31m \u001b[0m\u001b[36m\u001b[1mhint\u001b[0m\u001b[39m\u001b[1m:\u001b[0m Wheels are available for `\u001b[36mtriton\u001b[39m` (\u001b[36mv3.6.0\u001b[39m) on the following\n", + "\u001b[31m \u001b[0mplatforms: `\u001b[36mmanylinux_2_27_aarch64\u001b[39m`, `\u001b[36mmanylinux_2_27_x86_64\u001b[39m`,\n", + "\u001b[31m \u001b[0m`\u001b[36mmanylinux_2_28_aarch64\u001b[39m`, `\u001b[36mmanylinux_2_28_x86_64\u001b[39m`\n", + "\u001b[2mResolved \u001b[1m5 packages\u001b[0m \u001b[2min 495ms\u001b[0m\u001b[0m\n", + "\u001b[2mChecked \u001b[1m5 packages\u001b[0m \u001b[2min 1ms\u001b[0m\u001b[0m\n" + ] + } + ], + "source": [ + "import os, importlib.util\n", + "\n", + "!pip install --upgrade -qqq uv\n", + "if importlib.util.find_spec(\"torch\") is None or \"COLAB_\" in \"\".join(os.environ.keys()):\n", + " try:\n", + " import numpy\n", + "\n", + " get_numpy = f\"numpy=={numpy.__version__}\"\n", + " except:\n", + " get_numpy = \"numpy\"\n", + " !uv pip install -qqq \\\n", + " \"torch>=2.8.0\" \"triton==3.6.0\" {get_numpy} torchvision bitsandbytes \"transformers==4.56.2\" trackio \\\n", + " \"unsloth_zoo[base] @ git+https://github.com/unslothai/unsloth-zoo\" \\\n", + " \"unsloth[base] @ git+https://github.com/unslothai/unsloth\" \\\n", + " git+https://github.com/triton-lang/triton.git@0add68262ab0a2e33b84524346cb27cbb2787356#subdirectory=python/triton_kernels\n", + "elif importlib.util.find_spec(\"unsloth\") is None:\n", + " !uv pip install -qqq unsloth trackio\n", + "\n", + "!uv pip install --upgrade --no-deps transformers==4.56.2 tokenizers trl==0.22.2 unsloth unsloth_zoo" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "OjifwNNZ7bMx" + }, + "source": [ + "Next, we install the OpenEnv client and the connector for **OpenSpiel**—DeepMind's collection of game environments used in RL research. OpenSpiel includes implementations of classic games like Chess, Go, and our target: 2048." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "1yzMMSLR7dNj" + }, + "outputs": [], + "source": [ + "%%capture\n", + "!pip install -qqq openenv-core websockets\n", + "!pip install -qqq git+https://huggingface.co/spaces/openenv/openspiel_env" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "CcLYwLyQLADE" + }, + "source": [ + "## Loading OpenAI gpt-oss 20b\n", + "\n", + "We load the model with several memory optimizations:\n", + "\n", + "| Parameter | Value | Description |\n", + "|-----------|-------|-------------|\n", + "| `max_seq_length` | 768 | Maximum context length. Increase for longer outputs (uses more VRAM). |\n", + "| `load_in_4bit` | True | Quantizes weights to 4-bit, dramatically reducing memory usage. |\n", + "| `lora_rank` | 4 | LoRA adapter rank. Higher = more expressive but slower/more memory. |\n", + "| `offload_embedding` | True | Moves embeddings to CPU RAM, saving ~1GB VRAM. |" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 543, + "referenced_widgets": [ + "419e4369b36644d3abec159511a88ad8", + "eee0d2b6eb6047d8a4001f4999dcdc35", + "2d91f25a8d0b4dd39dc320b2cf17fc0b", + "1b84d7dc9567474c8587432d48342b71", + "f34cadfaf9bb46729aeeff9492ba9026", + "d257eaa588bd41fb947f81d306fe05cd", + "b09745899e1446129f397d822a21fc99", + "88aa58551344410a91b07af480d6ab53", + "4f38070c09354406928c5e7be6cff3fd", + "665f7b7e5496432985ed9f49829f5834", + "d16dc921c6554bc6b77abcb423721cc8", + "baf3db00d28f4c849bb6a6739e908c62", + "f4f2fb240a12406ab5e709be33b93683", + "4ca56d8605864a438290152268bfc686", + "de306a50594f463ba9c78c713bc33241", + "0f09a00375bd4d4c8863fc8fb7d64d61", + "89e3ad0d517f44d084df0d8a3ed40703", + "3f589cf3cb804ed29ba322e4fa10c511", + "2612008cf36949d9a6618a01ac817618", + "3e3ce50c437a412f9cc2bedb697648f7", + "fa5c42218fbb44378bef71551c3383e0", + "d82ab09313cc482f9b9b45192f489825", + "4e63263fc27b4f07b4f6abffba082379", + "eb13ad96565a44519e7cab9ce9483b90", + "0b851acfd32047bfb6bae17d43ccfcb1", + "d4c72002b5fe44d3ad7ae67f5536c889", + "a1e042e5b8ad4b028cc28ac54924207d", + "6d77c8dcb28240f7b860571d11f8b9af", + "4f29591af0d64d398515479b032a1b3d", + "667192c07a7340a9a72ed25648a0be64", + "aa714b1f70e8495b9299b51d6ac4c3c4", + "06e420cfa2974f7d8d7ec4b83f064a6e", + "3566e9058ebc45498b42e521f2314365", + "6af6de3285684e76b320571b44af5fc1", + "f7cc1614e13d4d22b83f787b20407a5f", + "072060f8bdb54a15baf838f67d376d99", + "5371b2fad7b04f97bd8f4671d844d2cb", + "b10d25fb43eb42198abf71c4e326bcff", + "7ce0f239a8514923b383f738bc0c9899", + "4bc16bd2399a43fe85967131af7f846a", + "29d98af49dd3412f84f5843b937029d1", + "93176d6b63284b4e832e0f028be90655", + "9c2371c32afe46519ee53427ea42bc9a", + "f9d4672fb86b4b4e9c3e9068dc479e5b", + "297bdff1add5414893319b185cb15da6", + "362947c7e102474cbf560f51e713bdff", + "7f63ff3e87aa4b86a4f8e64785d1d34c", + "cca7d922b85449a1b0c5c025b65fba10", + "5520c59f24cd414092bf5f952425611d", + "b1dcd2f908344949af01dab5a244e458", + "9e6f46ddb61943f4a168d70a209a7ddc", + "ae79b9a378ee4218beddf77ac9af6de7", + "e6d63cf58647443b9749195ec0579d87", + "e7ca6b7ba2094872a888da5511e2bb49", + "7c7d40163ecc4dae8a2c54af21de4661", + "4dc823fcd0dd4eaaa2e8aaff0daa9ad1", + "1b26eac03fed4c8784bd611474cf4607", + "a24e2cfbef794d35a0e22753352caa15", + "8368e4420e814c6f9be30994b69c66ee", + "eb8c197ce1fc41f78531e5e73ae14a89", + "b72fc22310714bf0bf6ff4021db5aba8", + "d2a7fa9dddc240e29330297870159c59", + "0e49035e0c3a4ee4ab477b475e74ef36", + "08d8f0cfd7614900a9c9bba888619749", + "d1cea390ecaa4583a698278fa3a438c9", + "5483233a9b224f3c8eb9337e4ed82314", + "cbed0dab2d7540f697eacec5a33e1061", + "dc9b2549ff834880ac19b578adfee5a5", + "48f65e25acd84b0cb582de66753215da", + "7c52841c7e714173bfd526b0a625bc9d", + "bd8f575034c041be93ff20f63388de2d", + "212c17e3829c4accb30265a3d9ee73dc", + "1daa373c890b4ae0a9cf6a3ec325693c", + "383e9b4b74e34cae96c1f46f41591b82", + "46cb3593b37c4cb9b4bac422bc5809c8", + "ae9728c04b974af29460ce5179a9edba", + "754f2452fbe14c7098215ec810ffbf14", + "3c2e00b9d20a4c9ea5b850b776752fd2", + "7c29e5f727ec4b608c06d0487a2c9e53", + "4beae5415d0647e2898a83313a08ea94", + "62cd4ccc430549e7a2c156222d47ebeb", + "4691273248984fdea29d83ec0a246cd9", + "ae3819cd042f43babef24199081bb97f", + "52e9a0ea76df4fd8822ccadadaadb501", + "a6a122064bc340868b5e0e11afa9c42f", + "3371a1da5d0e426bb6cc02a6c383dd6f", + "bedc42e908454f1494768194b43b2964", + "41ee8407344d402997b0574e0ae26c77", + "5d1e2fdbf7a2409abbafb63e4b160668", + "a4c19dc98fe943e09a26a60d23c8ff01", + "8f5799610318490492ff5aba76be3d1a", + "e78534b135d2465c83e3be614b71c8a4", + "2f57c8e713b94c8692837b4e17c9e983", + "f8d2164046fb46a298e2d80628808cb3", + "af23432e17664cd6852496e43f9de0cf", + "1055200185004ea2a95a05eb51232501", + "eb1068efb4364ee2893c4d21c58f38db", + "c4d592366499414a99f19bce7f0bd665", + "1ece5fe9597a4e3db2dd96c38995705c", + "050567dccb47456aaac65d118ac60a6b", + "ea2f3ca562444c46b50596a1d7cf9030", + "0b7287d482cc44dbb406d71f23f1aea0", + "474c7fe6ef4b430d9826171eded2ebf1", + "12a877304ddf45e49bbdfe056394c3d6", + "56b30cc150924879abfc138427f4ca98", + "02c88a690a384ae183c233b6927aaf57", + "d541fd264d7e4b2691e8efbd993e6ae7", + "cbc7bbfe983f4e6696f8dd6b37c50543", + "f290b17e3cd9487b9b97d54d6cec9efc", + "18d4cc3fd8ee4e3fbf27c574fd467f20", + "941d8cbf8188402eb603c18b6e979035", + "8708226010324a83b4f7900c3958d430", + "fc12c97d659640b3b2b2b48ad7f17e5b", + "aa22cb2d1dcf4001bd3720b075318906", + "bb358c5523814376a2d4690f88f20b74", + "4ba91b8d008e483d89f16f204326f6b6", + "31afdb187d2a44d8b3a101fa18543c13", + "28300c16023d4ad9a59784baea2f57aa", + "004f28173c3b4fb6a9f8c2068f5db81f", + "0b8fa4ff186a4bfeac18cf6d676e99df", + "ccb581e230604bb690015eb685e4b8e1" + ] + }, + "id": "DkIvEkIIkEyB", + "outputId": "feb2ac4d-19dc-4e55-ebbe-e28e0ef721c1" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "🦥 Unsloth: Will patch your computer to enable 2x faster free finetuning.\n", + "🦥 Unsloth Zoo will now patch everything to make training faster!\n", + "==((====))== Unsloth 2026.1.3: Fast Gpt_Oss patching. Transformers: 4.56.2.\n", + " \\\\ /| Tesla T4. Num GPUs = 1. Max memory: 14.741 GB. Platform: Linux.\n", + "O^O/ \\_/ \\ Torch: 2.9.0+cu126. CUDA: 7.5. CUDA Toolkit: 12.6. Triton: 3.5.0\n", + "\\ / Bfloat16 = FALSE. FA [Xformers = None. FA2 = False]\n", + " \"-____-\" Free license: http://github.com/unslothai/unsloth\n", + "Unsloth: Fast downloading is enabled - ignore downloading bars which are red colored!\n", + "Unsloth: Using float16 precision for gpt_oss won't work! Using float32.\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "419e4369b36644d3abec159511a88ad8", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "model.safetensors.index.json: 0.00B [00:00, ?B/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "baf3db00d28f4c849bb6a6739e908c62", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "model-00001-of-00004.safetensors: 0%| | 0.00/4.00G [00:00 0 ! Suggested 8, 16, 32, 64, 128\n", + " target_modules=[\n", + " \"q_proj\",\n", + " \"k_proj\",\n", + " \"v_proj\",\n", + " \"o_proj\",\n", + " \"gate_proj\",\n", + " \"up_proj\",\n", + " \"down_proj\",\n", + " ],\n", + " lora_alpha=lora_rank * 2, # *2 speeds up training\n", + " use_gradient_checkpointing=\"unsloth\", # Reduces memory usage\n", + " random_state=3407,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "N0QnO9_YJBOI" + }, + "source": [ + "## Connecting to the 2048 Game Environment\n", + "\n", + "OpenEnv lets us connect to game environments hosted remotely. We'll use a **Hugging Face Space** that runs the OpenSpiel 2048 game server. This architecture has several benefits:\n", + "\n", + "- **No local installation** of game dependencies (OpenSpiel can be tricky to build)\n", + "- **Consistent environment** across different machines\n", + "- **Scalable** - the same pattern works for more complex environments\n", + "\n", + "The Space exposes a WebSocket API that accepts actions and returns game states." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "hQrG81XV87-7" + }, + "outputs": [], + "source": [ + "from openspiel_env import OpenSpielEnv\n", + "from openspiel_env.models import OpenSpielAction, OpenSpielObservation" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "-WT0Zu0IcN_r" + }, + "source": [ + "The [openenv/openspiel_env](https://huggingface.co/spaces/openenv/openspiel_env) Space hosts a running OpenSpiel server configured for 2048. It handles game state management, validates moves, and returns observations after each action. You can also run the server locally for faster iteration:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "zBIye94T8djG" + }, + "outputs": [], + "source": [ + "# Connect to OpenSpiel 2048 environment on HuggingFace Spaces\n", + "# The game is configured server-side via OPENSPIEL_GAME=2048\n", + "OPENSPIEL_URL = \"https://openenv-openspiel-env.hf.space\"\n", + "# For local: OPENSPIEL_URL = \"http://localhost:8000\"\n", + "\n", + "env = OpenSpielEnv(base_url=OPENSPIEL_URL)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "P3rMiKLl9Ro2" + }, + "source": [ + "Let's see how the current 2048 game state looks like:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "uBPx0Hho9Xi1", + "outputId": "7e7fae62-6d83-4c5e-b8c5-b49a1ccc9c53" + }, + "outputs": [ + { + "data": { + "text/plain": [ + "OpenSpielObservation(done=False, reward=None, metadata={}, info_state=[0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0], legal_actions=[0, 1, 2], game_phase='initial', current_player_id=0, opponent_last_action=None)" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "result = env.reset()\n", + "current_state = result.observation\n", + "current_state" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "4Qz1tRVTAii8" + }, + "source": [ + "### Decoding the Game State\n", + "\n", + "OpenSpiel's 2048 `info_state` uses a compact encoding—not raw tile values. The first 16 elements represent the 4×4 board positions, with each value being **log₂ of the tile** (so 1 = 2¹ = 2, 2 = 2² = 4, etc.). Values of 0 represent empty cells.\n", + "\n", + "We need to:\n", + "1. Extract only the first 16 elements (the board)\n", + "2. Reshape into a 4×4 grid\n", + "3. Convert from log₂ encoding to actual tile values" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "7PVeoKW2AmKr", + "outputId": "5689ae76-cd68-48f5-e7e7-e05494109a63" + }, + "outputs": [ + { + "data": { + "text/plain": [ + "([[0, 1, 0, 0, 0, 0, 0, 0],\n", + " [0, 0, 0, 0, 0, 0, 0],\n", + " [0, 0, 0, 0, 0, 0, 0],\n", + " [0, 0, 0, 0, 0, 0, 0],\n", + " [0, 0, 0, 0, 0, 0, 0],\n", + " [0, 0, 0, 0, 0, 0, 0],\n", + " [0, 0, 0, 0, 1, 0, 0]],\n", + " 7)" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import numpy as np\n", + "\n", + "# 2048 game constants\n", + "BOARD_SIZE = 4\n", + "BOARD_CELLS = BOARD_SIZE * BOARD_SIZE # 16\n", + "WIN_TILE = 2048 # The target tile value to win\n", + "\n", + "\n", + "def convert_to_board(current_state):\n", + " \"\"\"\n", + " Convert OpenSpiel 2048 observation to a 4×4 board of tile values.\n", + "\n", + " OpenSpiel encodes tiles as log₂(value), so we convert back:\n", + " - 0 → 0 (empty)\n", + " - 1 → 2 (2^1)\n", + " - 2 → 4 (2^2)\n", + " - etc.\n", + " \"\"\"\n", + " # Extract only the first 16 elements (the board state)\n", + " raw_board = current_state.info_state[:BOARD_CELLS]\n", + "\n", + " # Convert from log₂ encoding to actual tile values\n", + " # 0 stays 0 (empty), otherwise 2^value\n", + " tiles = [int(2**val) if val > 0 else 0 for val in raw_board]\n", + "\n", + " # Reshape into 4×4 grid\n", + " board = [tiles[i * BOARD_SIZE : (i + 1) * BOARD_SIZE] for i in range(BOARD_SIZE)]\n", + " return board, BOARD_SIZE\n", + "\n", + "\n", + "convert_to_board(current_state)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "hCS56yu29dsG" + }, + "source": [ + "We also want to pretty print the game board! This is not entirely necessary, but it helps us visualize the game state and learn from the process." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "cellView": "form", + "id": "D9CI4jtgL5mw" + }, + "outputs": [], + "source": [ + "# @title (Collapsible) 2048 Game Renderer\n", + "def render_board(obs, colors: bool = True, border: bool = True, dot_for_zero: bool = True) -> str:\n", + " \"\"\"\n", + " Pretty-print the board with colors that scale from 0 up to self.target.\n", + " Uses ANSI 256-color codes (works in most terminals). Set colors=False to disable.\n", + " \"\"\"\n", + " import math\n", + "\n", + " b, size = convert_to_board(obs)\n", + " mx = max((max(row) for row in b), default=0)\n", + " cell_w = max(3, len(str(mx)))\n", + "\n", + " RESET = \"\\x1b[0m\"\n", + "\n", + " # A smooth-ish gradient from cool → warm\n", + " # (blue/cyan/green → yellow/orange/red). Tweak or expand as you like.\n", + " GRAD = [33, 39, 45, 51, 50, 49, 48, 47, 46, 82, 118, 154, 190, 226, 220, 214, 208, 202, 196]\n", + " ZERO_FG = 239 # dim gray\n", + "\n", + " def color_code(v: int) -> str:\n", + " if not colors:\n", + " return \"\"\n", + " if v == 0:\n", + " return f\"\\x1b[38;5;{ZERO_FG}m\"\n", + " # Normalize by exponent relative to target: r in [0,1]\n", + " t = max(2, WIN_TILE) # safety; avoid log2(1)\n", + " # Guard: if v is not a power of two or is <1, handle gracefully\n", + " try:\n", + " r = max(0.0, min(1.0, math.log2(v) / math.log2(t)))\n", + " except ValueError:\n", + " r = 0.0\n", + " idx = int(round(r * (len(GRAD) - 1)))\n", + " return f\"\\x1b[38;5;{GRAD[idx]}m\"\n", + "\n", + " def fmt(v: int) -> str:\n", + " s = \".\" if (v == 0 and dot_for_zero) else str(v)\n", + " s = s.rjust(cell_w)\n", + " return color_code(v) + s + (RESET if colors else \"\")\n", + "\n", + " def hline(left: str, mid: str, right: str) -> str:\n", + " return left + mid.join(\"─\" * cell_w for _ in range(size)) + right\n", + "\n", + " rows = []\n", + " if border:\n", + " rows.append(hline(\"┌\", \"┬\", \"┐\"))\n", + " for r in range(size):\n", + " content = \"│\".join(fmt(v) for v in b[r])\n", + " rows.append((\"│\" + content + \"│\") if border else content)\n", + " if border:\n", + " rows.append(\n", + " hline(\"└\" if r == size - 1 else \"├\", \"┴\" if r == size - 1 else \"┼\", \"┘\" if r == size - 1 else \"┤\")\n", + " )\n", + " return \"\\n\".join(rows)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "DAJE2LUo9oRR", + "outputId": "4b15a5e3-26fb-4ce0-a4b5-0261ab7a5999" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "┌───┬───┬───┬───┬───┬───┬───┐\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;33m 1\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;33m 1\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "└───┴───┴───┴───┴───┴───┴───┘\n" + ] + } + ], + "source": [ + "print(render_board(current_state))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "0AhUa4hW-Dji" + }, + "source": [ + "We can see the `legal_actions` ie what you can take as `[0, 1, 2, 3]` Let's try doing the action `0`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "b-gSgthFI_wq", + "outputId": "b490416b-e52e-4e13-d761-23c4efecf08a" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "┌───┬───┬───┬───┬───┬───┬───┐\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;33m 1\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;33m 1\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "└───┴───┴───┴───┴───┴───┴───┘\n" + ] + } + ], + "source": [ + "action = OpenSpielAction(action_id=0, game_name=\"2048\")\n", + "result = env.step(action)\n", + "current_state = result.observation\n", + "print(render_board(current_state))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "lPSNb8-A-iPn" + }, + "source": [ + "So it looks like `0` is a move up action! Let's try `1`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "IUel11Tc-oLB", + "outputId": "b67365e4-d760-4d49-dafc-57f4b397c98c" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "┌───┬───┬───┬───┬───┬───┬───┐\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;33m 1\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;33m 1\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "└───┴───┴───┴───┴───┴───┴───┘\n" + ] + } + ], + "source": [ + "action = OpenSpielAction(action_id=1, game_name=\"2048\")\n", + "result = env.step(action)\n", + "current_state = result.observation\n", + "print(render_board(current_state))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "nUlOshVe-qNL" + }, + "source": [ + "`1` is a move right action. And `2`:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "XU09-KA3-sqs", + "outputId": "f9089a3f-f564-418f-b2bd-512103f13135" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "┌───┬───┬───┬───┬───┬───┬───┐\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;33m 1\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;33m 1\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "└───┴───┴───┴───┴───┴───┴───┘\n" + ] + } + ], + "source": [ + "action = OpenSpielAction(action_id=2, game_name=\"2048\")\n", + "result = env.step(action)\n", + "current_state = result.observation\n", + "print(render_board(current_state))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "X2r7Zqw9-u-d" + }, + "source": [ + "`2` is a move down. And I guess `3` is just move left!" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "pFgspqn6-zd2", + "outputId": "65e800d7-9d78-420e-b4fa-232f4c919481" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "┌───┬───┬───┬───┬───┬───┬───┐\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;33m 1\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;33m 1\u001b[0m│\n", + "└───┴───┴───┴───┴───┴───┴───┘\n" + ] + } + ], + "source": [ + "action = OpenSpielAction(action_id=3, game_name=\"2048\")\n", + "result = env.step(action)\n", + "current_state = result.observation\n", + "print(render_board(current_state))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "RJP4TsDq-2ft" + }, + "source": [ + "We can also print the game status which indicates if no more moves are possible, and also the possible actions you can take!" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "MEa2ngmrvfNm", + "outputId": "48462391-11d8-4c55-e5c4-f05ec1df6dfe" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "False\n", + "[0, 1, 2]\n" + ] + } + ], + "source": [ + "print(current_state.done)\n", + "print(current_state.legal_actions)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "VR6czU96cpxf" + }, + "source": [ + "## RL Environment Setup: The Strategy Executor\n", + "\n", + "For reinforcement learning, we need a way to evaluate generated strategies. The key insight is that **our model doesn't play 2048 directly**—instead, it writes Python code that plays the game. We then execute that code and measure how well it performs.\n", + "\n", + "The `execute_strategy` function:\n", + "1. Takes a generated Python function (the \"strategy\")\n", + "2. Runs the 2048 game loop, calling the strategy for each move\n", + "3. Returns how many steps the game lasted and whether it reached 2048\n", + "\n", + "**Timeout protection**: LLM-generated code might contain infinite loops or be very slow. We wrap execution with a 2-second timeout to ensure the RL training loop doesn't hang." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "tdgjnf-8z_kr" + }, + "outputs": [], + "source": [ + "from typing import Callable\n", + "from unsloth import execute_with_time_limit\n", + "import itertools\n", + "\n", + "\n", + "def has_won(board) -> bool:\n", + " \"\"\"Check if the board contains a winning tile (2048 or higher).\"\"\"\n", + " max_tile = max(itertools.chain.from_iterable(board))\n", + " return max_tile >= WIN_TILE\n", + "\n", + "\n", + "def _execute_strategy(strategy, current_state: OpenSpielObservation):\n", + " \"\"\"Execute a strategy function on the 2048 game until completion or invalid move.\"\"\"\n", + " assert callable(strategy)\n", + "\n", + " steps = 0\n", + " total_reward = 0\n", + " board = None\n", + "\n", + " while not current_state.done:\n", + " board, _ = convert_to_board(current_state)\n", + " action = strategy(board)\n", + " try:\n", + " action = int(action)\n", + " except:\n", + " return steps, False\n", + " steps += 1\n", + "\n", + " # Invalid action - return current win status\n", + " if type(action) is not int or action not in current_state.legal_actions:\n", + " return steps, has_won(board) if board else False\n", + "\n", + " action = OpenSpielAction(action_id=action, game_name=\"2048\")\n", + " result = env.step(action)\n", + " current_state = result.observation\n", + " if result.reward is not None:\n", + " total_reward += result.reward\n", + "\n", + " # Game ended - check final board for win\n", + " if board is None:\n", + " board, _ = convert_to_board(current_state)\n", + " return steps, has_won(board)\n", + "\n", + "\n", + "@execute_with_time_limit(2)\n", + "def execute_strategy(strategy: Callable, current_state: OpenSpielObservation):\n", + " return _execute_strategy(strategy, current_state)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ywh0HizI9ayE" + }, + "source": [ + "Let's make a generic strategy to just hit `3`. We should expect this generic strategy to fail:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "5bkhqoZc0IO8", + "outputId": "c773dddf-7a5b-40fa-b061-33f407f8b804" + }, + "outputs": [ + { + "data": { + "text/plain": [ + "(1, False)" + ] + }, + "execution_count": 19, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "def always_move_left(board):\n", + " return 3\n", + "\n", + "\n", + "# Reset OpenEnv to an initial state!\n", + "result = env.reset()\n", + "current_state = result.observation\n", + "try:\n", + " steps, if_done = execute_strategy(always_move_left, current_state)\n", + "except TimeoutError as e:\n", + " print(f\"Timed out with error = {str(e)}\")\n", + "\n", + "steps, if_done" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "dkuHVdB09sgf" + }, + "source": [ + "To allow longer strategies for GPT-OSS Reinforcement Learning, we shall allow a 5 second timer." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "SK-LfzsA9wbW" + }, + "outputs": [], + "source": [ + "@execute_with_time_limit(5)\n", + "def execute_strategy(strategy: Callable, current_state: OpenSpielObservation):\n", + " return _execute_strategy(strategy, current_state)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "tRhLV_bZMYxy" + }, + "source": [ + "## Sandboxed Code Execution: Preventing Reward Hacking\n", + "\n", + "A critical challenge in RL with code generation is **reward hacking**—the model might learn to \"cheat\" rather than solve the actual problem. For example, it could:\n", + "\n", + "- Import external libraries to hardcode solutions\n", + "- Access global variables to manipulate game state directly \n", + "- Call system functions to bypass the game logic\n", + "\n", + "We use two safeguards:\n", + "\n", + "1. `check_python_modules` validates that the code only uses Python standard library imports (no numpy, pandas, etc.)\n", + "2. `create_locked_down_function` executes code in an isolated namespace with no access to global variables\n", + "\n", + "Let's see these in action. First, a valid strategy that only uses standard library:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "zz80kvg6M4BG", + "outputId": "19d7f047-127f-4667-8d8a-c64c132f87ea" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Only Python imports? True\n", + "{'stdlib': ['math', 'typing'], 'non_stdlib': [], 'relative_imports': 0}\n" + ] + } + ], + "source": [ + "from unsloth import check_python_modules\n", + "\n", + "sample = \"\"\"\n", + "def strategy(board):\n", + " import math\n", + " from typing import Callable\n", + " return \"0\"\n", + "\"\"\"\n", + "ok, info = check_python_modules(sample)\n", + "print(\"Only Python imports?\", ok)\n", + "print(info)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "bZzVWgKQ-VIg" + }, + "source": [ + "For the below piece of code, since we import `numpy`, we should not allow the execution:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "Z89Jw1KB-Ux7", + "outputId": "203c5dc5-bd09-42cc-b67f-08dd4a23af1c" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Only Python imports? False\n", + "{'stdlib': [], 'non_stdlib': ['numpy'], 'relative_imports': 0}\n" + ] + } + ], + "source": [ + "sample = \"\"\"\n", + "def strategy(board):\n", + " from numpy import matmul\n", + " return \"0\"\n", + "\"\"\"\n", + "ok, info = check_python_modules(sample)\n", + "print(\"Only Python imports?\", ok)\n", + "print(info)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "SDSrjOTLVyQm" + }, + "source": [ + "We also disallow global variable access. We'll use Unsloth's `create_locked_down_function` function\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "GcmYAmohVqw2", + "outputId": "b66e5e50-5ec8-42f6-8440-d5d8bc5dcc08" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "name 'np' is not defined\n" + ] + } + ], + "source": [ + "from unsloth import create_locked_down_function\n", + "\n", + "function = \"\"\"\n", + "def import_numpy():\n", + " np.matmul\n", + " print(\"Success\")\n", + "\"\"\"\n", + "f = create_locked_down_function(function)\n", + "try:\n", + " f()\n", + "except Exception as e:\n", + " print(str(e))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "5tJKwLUgZsRq", + "outputId": "512a7a58-5a11-4612-fccd-55ee2678f97c" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "60\n" + ] + } + ], + "source": [ + "from unsloth import create_locked_down_function\n", + "\n", + "function = \"\"\"\n", + "def add(a, b):\n", + " def adder(a):\n", + " return a + b\n", + " return adder(b) + b\n", + "\"\"\"\n", + "f = create_locked_down_function(function)\n", + "try:\n", + " print(f(10, 20))\n", + "except Exception as e:\n", + " print(str(e))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "8CzwCyXIPK04" + }, + "source": [ + "## Prompt Design: Instructing the Model\n", + "\n", + "The prompt is crucial—it tells the model what we expect it to generate. We want:\n", + "\n", + "1. A **single Python function** named `strategy(board)` that:\n", + " - Takes a 4×4 list of lists as input\n", + " - Returns a move: \"0\" (up), \"1\" (right), \"2\" (down), or \"3\" (left)\n", + "2. **Self-contained code** with all helpers defined inside the function\n", + "3. **Native Python only** (no external dependencies)\n", + "\n", + "This structured output format makes parsing straightforward and helps the model understand the task boundaries." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "B-2RRE4HMrQO", + "outputId": "20224710-abb6-4b34-a13f-dfb526c6fa9f" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Create a new short 2048 strategy using only native Python code.\n", + "You are given a list of list of numbers for the current board state.\n", + "Output one action for \"0\", \"1\", \"2\", \"3\" on what is the optimal next step.\n", + "Output your new short function in backticks using the format below:\n", + "```python\n", + "def strategy(board):\n", + " return \"0\" # Example\n", + "```\n", + "All helper functions should be inside def strategy. Only output the short function `strategy`.\n" + ] + } + ], + "source": [ + "prompt = \"\"\"\n", + "Create a new short 2048 strategy using only native Python code.\n", + "You are given a list of list of numbers for the current board state.\n", + "Output one action for \"0\", \"1\", \"2\", \"3\" on what is the optimal next step.\n", + "Output your new short function in backticks using the format below:\n", + "```python\n", + "def strategy(board):\n", + " return \"0\" # Example\n", + "```\n", + "All helper functions should be inside def strategy. Only output the short function `strategy`.\n", + "\"\"\".strip()\n", + "print(prompt)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "MIdudFUodN4i" + }, + "source": [ + "Let's see what the **base model** (before RL training) generates when given this prompt:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "9HJxrS76h3Ds", + "outputId": "b6ead093-ac19-49f6-9d0f-98e6e196dace" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "<|start|>system<|message|>You are ChatGPT, a large language model trained by OpenAI.\n", + "Knowledge cutoff: 2024-06\n", + "Current date: 2026-01-20\n", + "\n", + "Reasoning: low\n", + "\n", + "# Valid channels: analysis, commentary, final. Channel must be included for every message.\n", + "Calls to these tools must go to the commentary channel: 'functions'.<|end|><|start|>user<|message|>Create a new short 2048 strategy using only native Python code.\n", + "You are given a list of list of numbers for the current board state.\n", + "Output one action for \"0\", \"1\", \"2\", \"3\" on what is the optimal next step.\n", + "Output your new short function in backticks using the format below:\n", + "```python\n", + "def strategy(board):\n", + " return \"0\" # Example\n", + "```\n", + "All helper functions should be inside def strategy. Only output the short function `strategy`.<|end|><|start|>assistant<|channel|>analysis<|message|>We need to provide a short function. Probably simple heuristic: choose move with lowest collision? Use sum? Just a placeholder.<|end|><|start|>assistant<|channel|>final<|message|>```python\n", + "def strategy(board):\n", + " scores = [0]*4\n", + " for i in range(4):\n", + " r = [row[i] for row in board] # column\n", + " if any(r) and any(r) != 0:\n", + " # prefer moves that keep most tiles grouped\n", + " if i==0 or i==1:\n", + " scores[i] += sum(r)\n", + " best = scores.index(max(scores))\n", + " return str(best)\n", + "```<|return|>\n" + ] + } + ], + "source": [ + "text = tokenizer.apply_chat_template(\n", + " [{\"role\": \"user\", \"content\": prompt}],\n", + " tokenize=False,\n", + " add_generation_prompt=True,\n", + " reasoning_effort=\"low\",\n", + ")\n", + "\n", + "from transformers import TextStreamer\n", + "\n", + "_ = model.generate(\n", + " **tokenizer(text, return_tensors=\"pt\").to(\"cuda\"),\n", + " temperature=1.0,\n", + " max_new_tokens=512,\n", + " streamer=TextStreamer(tokenizer, skip_prompt=False),\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "iknaWZNudTNq" + }, + "source": [ + "## Designing Reward Functions\n", + "\n", + "Reward functions are the heart of RL—they define what the \"good\" behavior that we want to encourage in the model looks like. For code generation, we need a multi-part reward that captures different aspects of quality:\n", + "\n", + "| Reward Function | Purpose | Score Range |\n", + "|-----------------|---------|-------------|\n", + "| `function_works` | Is the generated code syntactically valid and executable? | -2.0 to +1.0 |\n", + "| `no_cheating` | Does the code avoid forbidden imports (numpy, etc.)? | -20.0 to +1.0 |\n", + "| `strategy_succeeds` | Does the strategy actually play 2048 well? | -3.0 to +20.0 |\n", + "\n", + "\n", + "Let's break down the reward functions into some practical examples:\n", + "- We could heavily penalize cheating (-20.0) to make honest solutions more rewarding.\n", + "- We could massively reward success (+20.0) since reaching 2048 is rare initially.\n", + "- We could graduated penalties for partial failures (timeout vs. crash vs. invalid syntax). This gives the model more information to learn from, creating a 'richer' reward signal.\n", + "\n", + "First, we need a helper to extract the Python function from the model's markdown-formatted output:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "8JJGXKdJ-Zl_", + "outputId": "1474af23-0d1f-4e73-b111-579ea0805d27" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "def strategy(board):\n", + " return \"0\" # Example\n" + ] + } + ], + "source": [ + "def extract_function(text):\n", + " if text.count(\"```\") >= 2:\n", + " first = text.find(\"```\") + 3\n", + " second = text.find(\"```\", first)\n", + " fx = text[first:second].strip()\n", + " fx = fx.removeprefix(\"python\\n\")\n", + " fx = fx[fx.find(\"def\") :]\n", + " if fx.startswith(\"def strategy(board):\"):\n", + " return fx\n", + " return None\n", + "\n", + "\n", + "print(extract_function(prompt))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "KLXEcf_HSJlI" + }, + "source": [ + "Below is our `function_works` reward function which uses Python's `exec` but guarded by not allowing leakage of local and global variables. We can also use `check_python_modules` first to check if there are errors before even executing the function:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "h3-B0IIsS56S", + "outputId": "33348b7f-c5dd-4d22-a0ac-e68a2264426f" + }, + "outputs": [ + { + "data": { + "text/plain": [ + "(False,\n", + " {'error': \"SyntaxError: expected '(' (, line 1)\",\n", + " 'stdlib': [],\n", + " 'non_stdlib': [],\n", + " 'relative_imports': 0})" + ] + }, + "execution_count": 28, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "ok, info = check_python_modules(\"def a\")\n", + "ok, info" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "qgFNXORy-lpO" + }, + "outputs": [], + "source": [ + "def function_works(completions, **kwargs):\n", + " scores = []\n", + " for completion in completions:\n", + " score = 0\n", + " response = completion[0][\"content\"]\n", + " function = extract_function(response)\n", + " if function is not None:\n", + " ok, info = check_python_modules(function)\n", + " if function is None or \"error\" in info:\n", + " score = -2.0\n", + " else:\n", + " try:\n", + " new_strategy = create_locked_down_function(function)\n", + " score = 1.0\n", + " except:\n", + " score = -0.5\n", + " scores.append(score)\n", + " return scores" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Gf69i2WT-m4K" + }, + "source": [ + "`no_cheating` checks if the function cheated since it might have imported Numpy or other functions:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "cUfHzCVx-nGK" + }, + "outputs": [], + "source": [ + "def no_cheating(completions, **kwargs):\n", + " scores = []\n", + " for completion in completions:\n", + " score = 0\n", + " response = completion[0][\"content\"]\n", + " function = extract_function(response)\n", + " if function is not None:\n", + " ok, info = check_python_modules(function)\n", + " scores.append(1.0 if ok else -20.0) # Penalize heavily!\n", + " else:\n", + " scores.append(-1.0) # Failed creating function\n", + " return scores" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "slnqWG3FTror" + }, + "source": [ + "Next `strategy_succeeds` checks if the strategy actually allows the game to terminate. Imagine if the strategy simply returned \"0\" which would fail after a time limit of 10 seconds.\n", + "\n", + "We also add a global `PRINTER` to print out the strategy and board state." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "sNi129lYTpZ2" + }, + "outputs": [], + "source": [ + "import numpy as np\n", + "\n", + "global PRINTER\n", + "PRINTER = 0\n", + "\n", + "\n", + "def strategy_succeeds(completions, **kwargs):\n", + " global PRINTER\n", + " scores = []\n", + " for completion in completions:\n", + " printed = False\n", + " score = 0\n", + " response = completion[0][\"content\"]\n", + " function = extract_function(response)\n", + " if PRINTER % 5 == 0:\n", + " printed = True\n", + " print(function)\n", + " PRINTER += 1\n", + " if function is not None:\n", + " ok, info = check_python_modules(function)\n", + " if function is None or \"error\" in info:\n", + " scores.append(0)\n", + " continue\n", + " try:\n", + " new_strategy = create_locked_down_function(function)\n", + " except:\n", + " scores.append(0)\n", + " continue\n", + " try:\n", + " # Reset OpenEnv to an initial state!\n", + " result = env.reset()\n", + " current_state = result.observation\n", + " steps, if_done = execute_strategy(new_strategy, current_state)\n", + " print(f\"Steps = {steps} If Done = {if_done}\")\n", + " if printed is False:\n", + " print(function)\n", + " print(render_board(current_state))\n", + " if if_done:\n", + " scores.append(20.0) # Success - massively reward!\n", + " else:\n", + " scores.append(2.0) # Failed but function works!\n", + " except TimeoutError as e:\n", + " print(\"Timeout\")\n", + " scores.append(-1.0) # Failed with timeout\n", + " except Exception as e:\n", + " print(f\"Exception = {str(e)}\")\n", + " scores.append(-3.0) # Failed\n", + " return scores" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "TCpSxtvSeAG_" + }, + "source": [ + "We'll now create the dataset which includes a replica of our prompt. Remember to add a reasoning effort of low! You can choose high reasoning mode, but this'll only work on more memory GPUs like MI300s." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "Ldf6SjLHVPRv", + "outputId": "684f36b7-d23a-4229-afd6-b0033edf7962" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "181\n" + ] + }, + { + "data": { + "text/plain": [ + "{'prompt': [{'content': 'Create a new short 2048 strategy using only native Python code.\\nYou are given a list of list of numbers for the current board state.\\nOutput one action for \"0\", \"1\", \"2\", \"3\" on what is the optimal next step.\\nOutput your new short function in backticks using the format below:\\n```python\\ndef strategy(board):\\n return \"0\" # Example\\n```\\nAll helper functions should be inside def strategy. Only output the short function `strategy`.',\n", + " 'role': 'user'}],\n", + " 'answer': 0,\n", + " 'reasoning_effort': 'low'}" + ] + }, + "execution_count": 32, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from datasets import Dataset\n", + "\n", + "dataset = Dataset.from_list(\n", + " [{\"prompt\": [{\"role\": \"user\", \"content\": prompt.strip()}], \"answer\": 0, \"reasoning_effort\": \"low\"}] * 1000\n", + ")\n", + "maximum_length = len(\n", + " tokenizer.apply_chat_template([{\"role\": \"user\", \"content\": prompt.strip()}], add_generation_prompt=True)\n", + ")\n", + "print(maximum_length)\n", + "dataset[0]" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "9-IOMhVg-2AM" + }, + "source": [ + "## Training with GRPO\n", + "\n", + "**Group Relative Policy Optimization (GRPO)** is an RL algorithm designed for language models. Unlike PPO which requires a separate value network, GRPO computes advantages by comparing generations within the same prompt group—making it simpler and more memory-efficient.\n", + "\n", + "Key training parameters:\n", + "- `num_generations=2`: Generate 2 candidates per prompt to compute relative rewards\n", + "- `max_steps=600`: Total training steps (~5 hours on T4)\n", + "- `temperature=1.0`: Controls generation randomness (higher = more exploration)\n", + "\n", + "We use [TrackIO](https://github.com/gradio-app/trackio) for live visualization of training metrics directly in the notebook." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "ptqkXK2D4d6p", + "outputId": "42f0f57e-d1a8-4a21-8834-c2dca0e493fb" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Unsloth: We now expect `per_device_train_batch_size` * `gradient_accumulation_steps` * `world_size` to be a multiple of `num_generations`.\n", + "We will change the batch size of 1 to the `num_generations` of 2\n" + ] + } + ], + "source": [ + "max_prompt_length = maximum_length + 1 # + 1 just in case!\n", + "max_completion_length = max_seq_length - max_prompt_length\n", + "\n", + "from trl import GRPOConfig, GRPOTrainer\n", + "\n", + "training_args = GRPOConfig(\n", + " temperature=1.0,\n", + " learning_rate=2e-4,\n", + " weight_decay=0.001,\n", + " warmup_ratio=0.1,\n", + " lr_scheduler_type=\"linear\",\n", + " optim=\"adamw_8bit\",\n", + " logging_steps=1,\n", + " per_device_train_batch_size=1,\n", + " gradient_accumulation_steps=1, # Increase to 4 for smoother training\n", + " num_generations=2, # Decrease if out of memory\n", + " max_prompt_length=max_prompt_length,\n", + " max_completion_length=max_completion_length,\n", + " # num_train_epochs = 1, # Set to 1 for a full training run\n", + " max_steps=600,\n", + " save_steps=100,\n", + " report_to=\"trackio\", # Can use Weights & Biases, TrackIO\n", + " output_dir=\"outputs\",\n", + " # For optional training + evaluation\n", + " # fp16_full_eval = True,\n", + " # per_device_eval_batch_size = 4,\n", + " # eval_accumulation_steps = 1,\n", + " # eval_strategy = \"steps\",\n", + " # eval_steps = 1,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "r9Mv8UZO5hz-" + }, + "source": [ + "And let's run the trainer! If you scroll up, you'll see a table of rewards. The goal is to see the `reward` column increase!\n", + "\n", + "You might have to wait 150 to 200 steps for any action. You'll probably get 0 reward for the first 100 steps. Please be patient!\n", + "\n", + "| Step | Training Loss | reward | reward_std | completion_length | kl |\n", + "|------|---------------|-----------|------------|-------------------|----------|\n", + "| 1 | 0.000000 | 0.125000 | 0.000000 | 200.000000 | 0.000000 |\n", + "| 2 | 0.000000 | 0.072375 | 0.248112 | 200.000000 | 0.000000 |\n", + "| 3 | 0.000000 | -0.079000 | 0.163776 | 182.500000 | 0.000005 |\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "vzOuSVCL_GA9", + "outputId": "880e7b31-fd7b-4ddc-96c6-9661a6e9a85e" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Unsloth: Switching to float32 training since model cannot work with float16\n" + ] + } + ], + "source": [ + "trainer = GRPOTrainer(\n", + " model=model,\n", + " processing_class=tokenizer,\n", + " reward_funcs=[\n", + " function_works,\n", + " no_cheating,\n", + " strategy_succeeds,\n", + " ],\n", + " args=training_args,\n", + " train_dataset=dataset,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "fQhtuwP4cf34" + }, + "source": [ + "And let's train the model! **NOTE** This might be quite slow! 600 steps takes ~5 hours or longer.\n", + "\n", + "[TrackIO](https://github.com/gradio-app/trackio) might be a bit slow to load - wait 2 minutes until the graphs pop up!" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000 + }, + "id": "VGRxPdSCcfC3", + "outputId": "9fb52ba6-2b05-4cdf-cd86-642302a96dde" + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "==((====))== Unsloth - 2x faster free finetuning | Num GPUs used = 2\n", + " \\\\ /| Num examples = 1,000 | Num Epochs = 1 | Total steps = 600\n", + "O^O/ \\_/ \\ Batch size per device = 2 | Gradient accumulation steps = 1\n", + "\\ / Data Parallel GPUs = 1 | Total batch size (2 x 1 x 1) = 2\n", + " \"-____-\" Trainable parameters = 1,990,656 of 20,916,747,840 (0.01% trained)\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "* Running on public URL: https://68c7c4f7168a6e9cd6.gradio.live\n", + "* Trackio project initialized: huggingface\n", + "* Trackio metrics logged to: /root/.cache/huggingface/trackio\n" + ] + }, + { + "data": { + "text/html": [ + "
" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "* GPU detected, enabling automatic GPU metrics logging\n", + "* Created new run: dainty-sunset-0\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "`generation_config` default values have been modified to match model-specific defaults: {'max_length': 131072}. If this is not desired, please set these values explicitly.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "def strategy(board):\n", + " # simple look‑ahead: pick the move that keeps the board almost sorted\n", + " # score is total immobility (higher=better)\n", + " def score(b):\n", + " s = 0\n", + " n = len(b)\n", + " for i in range(n):\n", + " for j in range(n):\n", + " v = b[i][j]\n", + " if v != 0:\n", + " # neighbors that can merge\n", + " for di,dj in [(1,0),(-1,0),(0,1),(0,-1)]:\n", + " ni, nj = i+di, j+dj\n", + " if 0 <= ni < n and 0 <= nj < n:\n", + " if b[ni][nj] == v:\n", + " s += v\n", + " return s\n", + " moves = []\n", + " for m in [\"0\",\"1\",\"2\",\"3\"]:\n", + " new_b = [row[:] for row in board]\n", + " # simulate move (simplified: just skip actual moving)\n", + " # Here we just pick the move with highest score (mock)\n", + " moves.append((score(new_b), m))\n", + " best = max(moves)[1]\n", + " return best\n", + "Steps = 1 If Done = False\n", + "┌───┬───┬───┬───┬───┬───┬───┐\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;33m 1\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;33m 1\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "└───┴───┴───┴───┴───┴───┴───┘\n", + "Steps = 9 If Done = False\n", + "def strategy(board):\n", + " def apply(board, d):\n", + " size = len(board)\n", + " moved, new = False, [[0]*size for _ in range(size)]\n", + " for i in range(size):\n", + " row = board[i] if d in (0,1) else [board[j][i] for j in range(size)]\n", + " vals = [v for v in row if v]\n", + " merged = []\n", + " j = 0\n", + " while j < len(vals):\n", + " if j+1 < len(vals) and vals[j]==vals[j+1]:\n", + " merged.append(vals[j]*2)\n", + " j+=2\n", + " else:\n", + " merged.append(vals[j]); j+=1\n", + " merged += [0]*(size-len(merged))\n", + " if d==0: new[i]=merged\n", + " if d==1: new[i]=merged[::-1]\n", + " if d==2: new[i]=merged\n", + " if d==3: new[i]=merged[::-1]\n", + " return new\n", + " best, best_move = 0, \"0\"\n", + " for move in \"0123\":\n", + " new = apply(board, int(move))\n", + " cnt = sum(1 for r in new for v in r if v==0)\n", + " if cnt > best:\n", + " best, best_move = cnt, move\n", + " return best_move\n", + "┌───┬───┬───┬───┬───┬───┬───┐\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;33m 1\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;33m 1\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "└───┴───┴───┴───┴───┴───┴───┘\n", + "Unsloth: Will smartly offload gradients to save VRAM!\n" + ] + }, + { + "data": { + "text/html": [ + "\n", + "
\n", + " \n", + " \n", + " [ 15/600 1:09:12 < 51:54:20, 0.00 it/s, Epoch 0.01/1]\n", + "
\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
StepTraining Lossrewardreward_stdcompletions / mean_lengthcompletions / min_lengthcompletions / max_lengthcompletions / clipped_ratiocompletions / mean_terminated_lengthcompletions / min_terminated_lengthcompletions / max_terminated_lengthklrewards / function_works / meanrewards / function_works / stdrewards / no_cheating / meanrewards / no_cheating / stdrewards / strategy_succeeds / meanrewards / strategy_succeeds / std
10.0000004.0000000.000000312.000000278.000000346.0000000.000000312.000000278.000000346.0000000.0001031.0000000.0000001.0000000.0000002.0000000.000000
20.0000004.0000000.000000340.000000338.000000342.0000000.000000340.000000338.000000342.0000000.0000551.0000000.0000001.0000000.0000002.0000000.000000
30.000000-1.2500002.474874557.000000528.000000586.0000000.500000528.000000528.000000528.0000000.000063-1.2500001.0606600.0000001.4142140.0000000.000000
40.0000004.0000000.00000082.00000052.000000112.0000000.00000082.00000052.000000112.0000000.0001751.0000000.0000001.0000000.0000002.0000000.000000
50.0000004.0000000.000000355.500000244.000000467.0000000.000000355.500000244.000000467.0000000.0015421.0000000.0000001.0000000.0000002.0000000.000000
60.0000004.0000000.000000278.000000142.000000414.0000000.000000278.000000142.000000414.0000000.0081321.0000000.0000001.0000000.0000002.0000000.000000
70.0000004.0000000.000000370.000000183.000000557.0000000.000000370.000000183.000000557.0000000.0073501.0000000.0000001.0000000.0000002.0000000.000000
80.0000004.0000000.000000317.500000190.000000445.0000000.000000317.500000190.000000445.0000000.0123451.0000000.0000001.0000000.0000002.0000000.000000
90.0000000.5000004.949748321.50000057.000000586.0000000.50000057.00000057.00000057.0000000.018377-0.5000002.1213200.0000001.4142141.0000001.414214
100.0000000.5000004.949748407.500000229.000000586.0000000.500000229.000000229.000000229.0000000.031208-0.5000002.1213200.0000001.4142141.0000001.414214
110.0000004.0000000.000000225.500000187.000000264.0000000.000000225.500000187.000000264.0000000.0375501.0000000.0000001.0000000.0000002.0000000.000000
120.000000-3.0000000.000000586.000000586.000000586.0000001.0000000.0000000.0000000.0000000.000167-2.0000000.000000-1.0000000.0000000.0000000.000000
130.000100-2.0000001.414214491.000000396.000000586.0000000.500000396.000000396.000000396.0000000.129521-0.5000002.1213200.0000001.414214-1.5000002.121320

" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Steps = 1 If Done = False\n", + "def strategy(board):\n", + " # Count empty cells for each move\n", + " def score_for(move):\n", + " temp = [row[:] for row in board]\n", + " # simulate move\n", + " for i in range(4):\n", + " line = [temp[j][i] for j in range(4)] if move==3 else [temp[i][j] for j in range(4)]\n", + " # shift and merge\n", + " new_line = []\n", + " merged = False\n", + " for val in (line if move in (0,2) else reversed(line)):\n", + " if val == 0: continue\n", + " if new_line and new_line[-1] == val and not merged:\n", + " new_line[-1] *= 2\n", + " merged = True\n", + " else:\n", + " new_line.append(val)\n", + " # fill rest with zeros\n", + " new_line += [0]*(4-len(new_line))\n", + " if move==3:\n", + " for j in range(4): temp[j][i] = new_line[j]\n", + " else:\n", + " for j in range(4): temp[i][j] = new_line[j]\n", + " return sum(new_line) # simple heuristic\n", + " best = -1; best_move=None\n", + " for m in range(4):\n", + " if score_for(m) > best:\n", + " best = score_for(m); best_move=str(m)\n", + " return best_move\n", + "┌───┬───┬───┬───┬───┬───┬───┐\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;33m 1\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;33m 1\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "└───┴───┴───┴───┴───┴───┴───┘\n", + "Steps = 9 If Done = False\n", + "def strategy(board):\n", + " # Assign scores to each move based on total number of merges and minimal tile movement\n", + " scores = {}\n", + " dirs = [(0,1),(0,-1),(1,0),(-1,0)] # right, left, down, up\n", + " # evaluate each direction\n", + " for i, (dx, dy) in enumerate(dirs):\n", + " tmp = [row[:] for row in board] # copy\n", + " for y in range(len(tmp)):\n", + " line = tmp[y] if dx==0 else [tmp[x][y] for x in range(len(tmp))]\n", + " # slide and combine\n", + " new_line = [v for v in line if v!=0]\n", + " for k in range(len(new_line)-1):\n", + " if new_line[k]==new_line[k+1]:\n", + " new_line[k]*=2\n", + " new_line.pop(k+1)\n", + " new_line+= [0]*(len(line)-len(new_line))\n", + " # place back\n", + " if dx==0:\n", + " tmp[y] = new_line\n", + " else:\n", + " for x in range(len(tmp)):\n", + " tmp[x][y] = new_line[x]\n", + " # score by number of zero tiles (more space)\n", + " scores[i] = sum(v==0 for row in tmp for v in row)\n", + " # choose move with most empty tiles\n", + " best = max(scores, key=scores.get)\n", + " return str(best)\n", + "┌───┬───┬───┬───┬───┬───┬───┐\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;33m 1\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;33m 1\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "└───┴───┴───┴───┴───┴───┴───┘\n", + "def strategy(board):\n", + " def move(board, dir):\n", + " size = len(board)\n", + " def compress(line):\n", + " nonlocal line\n", + " line = [x for x in line if x]\n", + " for i in range(len(line)-1):\n", + " if line[i]==line[i+1]:\n", + " line[i]*=2\n", + " line[i+1]=0\n", + " line=[x for x in line if x]\n", + " return line+[0]*(size-len(line))\n", + " if dir==0: # up\n", + " res=[[0]*size for _ in range(size)]\n", + " for c in range(size):\n", + " col=[board[r][c] for r in range(size)]\n", + " col=compress(col)\n", + " for r in range(size):\n", + " res[r][c]=col[r]\n", + " return res\n", + " if dir==1: # down\n", + " res=[[0]*size for _ in range(size)]\n", + " for c in range(size):\n", + " col=[board[r][c] for r in range(size)][::-1]\n", + " col=compress(col)\n", + " col=col[::-1]\n", + " for r in range(size):\n", + " res[r][c]=col[r]\n", + " return res\n", + " if dir==2: # left\n", + " res=[[0]*size for _ in range(size)]\n", + " for r in range(size):\n", + " line=board[r]\n", + " line=compress(line)\n", + " res[r]=line\n", + " return res\n", + " if dir==3: # right\n", + " res=[[0]*size for _ in range(size)]\n", + " for r in range(size):\n", + " line=board[r][::-1]\n", + " line=compress(line)\n", + " line=line[::-1]\n", + " res[r]=line\n", + " return res\n", + " best=None\n", + " best_sum=-1\n", + " for d in range(4):\n", + " new=move(board,d)\n", + " if new==board: continue\n", + " s=sum(sum(row) for row in new)\n", + " if s>best_sum:\n", + " best_sum=s; best=str(d)\n", + " return best if best is not None else \"0\"\n", + "Steps = 9 If Done = False\n", + "def strategy(board):\n", + " import random\n", + " best = None\n", + " best_score = -1\n", + " for move in '0123':\n", + " b = board\n", + " # simulate move by simple shift (not full 2048 logic)\n", + " # This is a placeholder: choose random valid move\n", + " if random.random() < 0.5:\n", + " return move\n", + " return best if best else \"0\"\n", + "┌───┬───┬───┬───┬───┬───┬───┐\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;33m 1\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;33m 1\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "└───┴───┴───┴───┴───┴───┴───┘\n", + "Steps = 9 If Done = False\n", + "def strategy(board):\n", + " # Very simple strategy: always return the first possible direction (0).\n", + " return \"0\"\n", + "┌───┬───┬───┬───┬───┬───┬───┐\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;33m 1\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;33m 1\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "└───┴───┴───┴───┴───┴───┴───┘\n", + "Steps = 9 If Done = False\n", + "def strategy(board):\n", + " #\n", + " # Try to reduce the number of empty tiles by making a merge first.\n", + " #\n", + " def can_merge(up, down):\n", + " return any((r, c) for r in range(4) for c in range(4)\n", + " if board[r][c] == 0 and board[down(r)][down(c)] == r and board[up(r)][up(c)] == r)\n", + " #\n", + " # Prefer to move towards the corner that is most populated.\n", + " #\n", + " if any(board[0][c] == 0 for c in range(4)):\n", + " return \"0\" # up\n", + " if any(board[3][c] == 0 for c in range(4)):\n", + " return \"1\" # down\n", + " if any(board[r][0] == 0 for r in range(4)):\n", + " return \"2\" # left\n", + " return \"3\" # right\n", + "┌───┬───┬───┬───┬───┬───┬───┐\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;33m 1\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;33m 1\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "└───┴───┴───┴───┴───┴───┴───┘\n", + "Steps = 1 If Done = False\n", + "def strategy(board):\n", + " # Simple heuristic: choose the move that results in the highest number of empty cells after a move\n", + " moves = []\n", + " for move in range(4):\n", + " new_board = [row[:] for row in board]\n", + " # simulate move\n", + " def compress_and_merge(line):\n", + " filtered = [x for x in line if x != 0]\n", + " merged = []\n", + " skip = False\n", + " for i, val in enumerate(filtered):\n", + " if skip:\n", + " skip = False\n", + " continue\n", + " if i + 1 < len(filtered) and filtered[i] == filtered[i+1]:\n", + " merged.append(val * 2)\n", + " skip = True\n", + " else:\n", + " merged.append(val)\n", + " merged += [0] * (len(line) - len(merged))\n", + " return merged\n", + " if move == 0: # left\n", + " for r in range(4):\n", + " new_board[r] = compress_and_merge(new_board[r])\n", + " elif move == 1: # right\n", + " for r in range(4):\n", + " new_board[r] = list(reversed(compress_and_merge(list(reversed(new_board[r])))))\n", + " elif move == 2: # up\n", + " for c in range(4):\n", + " col = [new_board[r][c] for r in range(4)]\n", + " merged = compress_and_merge(col)\n", + " for r in range(4):\n", + " new_board[r][c] = merged[r]\n", + " elif move == 3: # down\n", + " for c in range(4):\n", + " col = [new_board[r][c] for r in range(4)]\n", + " merged = list(reversed(compress_and_merge(list(reversed(col)))))\n", + " for r in range(4):\n", + " new_board[r][c] = merged[r]\n", + " moves.append((sum(row.count(0) for row in new_board), move))\n", + " return str(max(moves)[1])\n", + "┌───┬───┬───┬───┬───┬───┬───┐\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;33m 1\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;33m 1\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "└───┴───┴───┴───┴───┴───┴───┘\n", + "def strategy(board):\n", + " # compute all possible moves and pick one with max score (simple heuristic)\n", + " best, best_val = None, -1\n", + " for move in map(str, range(4)):\n", + " # simulate move\n", + " new_board = [row[:] for row in board]\n", + " # apply move logic (omitted for brevity)\n", + " # evaluate board\n", + " val = sum(sum(row) for row in new_board) # placeholder\n", + " if val > best_val:\n", + " best_val, best = val, move\n", + " return best\n", + "Steps = 9 If Done = False\n", + "┌───┬───┬───┬───┬───┬───┬───┐\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;33m 1\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;33m 1\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "└───┴───┴───┴───┴───┴───┴───┘\n", + "Steps = 9 If Done = False\n", + "def strategy(board):\n", + " # simple heuristic: choose first direction that merges a pair\n", + " for d in range(4):\n", + " visited = set()\n", + " for i in range(4):\n", + " for j in range(4):\n", + " if board[i][j] == 0:\n", + " continue\n", + " ni, nj = i, j\n", + " if d == 0: # up\n", + " ni -= 1\n", + " elif d == 1: # down\n", + " ni += 1\n", + " elif d == 2: # left\n", + " nj -= 1\n", + " else: # right\n", + " nj += 1\n", + " if 0 <= ni < 4 and 0 <= nj < 4:\n", + " if board[ni][nj] == board[i][j]:\n", + " return str(d)\n", + " return \"0\"\n", + "┌───┬───┬───┬───┬───┬───┬───┐\n", + "│\u001b[38;5;33m 1\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;33m 1\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "└───┴───┴───┴───┴───┴───┴───┘\n", + "Steps = 9 If Done = False\n", + "def strategy(board):\n", + " # look for the best next move using a simple heuristic\n", + " max_val = -1\n", + " move = \"0\"\n", + " for m in (\"0\",\"1\",\"2\",\"3\"):\n", + " # copy board\n", + " import copy\n", + " b = copy.deepcopy(board)\n", + " # simulate move\n", + " r=False\n", + " if m==\"0\": # up\n", + " for c in range(len(b)):\n", + " col=[b[r][c] for r in range(len(b))]\n", + " col=[n for n in col if n]\n", + " i=0\n", + " while i< len(col)-1:\n", + " if col[i]==col[i+1]:\n", + " col[i]*=2; del col[i+1]; i+=1\n", + " i+=1\n", + " for r in range(len(b)):\n", + " b[r][c]=col[r] if r0:\n", + " if col[i]==col[i-1]:\n", + " col[i]*=2; del col[i-1]; i-=1\n", + " i-=1\n", + " for r in range(len(b)):\n", + " b[r][c]=col[len(col)-1-r] if r0:\n", + " if row[i]==row[i-1]:\n", + " row[i]*=2; del row[i-1]; i-=1\n", + " i-=1\n", + " b[r]=[0]*(len(b)-len(row))+row\n", + " # evaluate\n", + " val=sum(max(row) for row in b)\n", + " if val>max_val:\n", + " max_val=val; move=m\n", + " return move\n", + "┌───┬───┬───┬───┬───┬───┬───┐\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;33m 1\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;33m 1\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "└───┴───┴───┴───┴───┴───┴───┘\n", + "Steps = 0 If Done = False\n", + "def strategy(board):\n", + " # Count tiles in entire board\n", + " total = sum(sum(row) for row in board)\n", + " if total == 0: # no tiles\n", + " return \"0\"\n", + " # Heuristic: prefer moving up if average value of upper row > lower row\n", + " upper = sum(board[0])\n", + " lower = sum(board[-1])\n", + " left = sum(row[0] for row in board)\n", + " right = sum(row[-1] for row in board)\n", + " moves = [(upper - lower, \"0\"), (right - left, \"1\")], \n", + " # pick the move with biggest difference (push bigger numbers up or right)\n", + " best_move = max(moves)[1]\n", + " return best_move\n", + "┌───┬───┬───┬───┬───┬───┬───┐\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;33m 1\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;33m 1\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "└───┴───┴───┴───┴───┴───┴───┘\n", + "Steps = 9 If Done = False\n", + "def strategy(board):\n", + " # Possible moves: 0=up, 1=right, 2=down, 3=left\n", + " best_score = -1\n", + " best_move = 0\n", + " dirs = [(0, -1), (1, 0), (0, 1), (-1, 0)] # mapping: up, right, down, left\n", + " for move, (dx, dy) in enumerate(dirs):\n", + " new_board = [row[:] for row in board]\n", + " moved = False\n", + " for x in range(4):\n", + " for y in range(4):\n", + " if dx != 0:\n", + " nx, ny = x + dx, y\n", + " else:\n", + " nx, ny = x, y + dy\n", + " if 0 <= nx < 4 and 0 <= ny < 4:\n", + " if board[x][y] != 0 and new_board[nx][ny] == 0:\n", + " new_board[nx][ny] = board[x][y]\n", + " new_board[x][y] = 0\n", + " moved = True\n", + " # Merge\n", + " if dx != 0:\n", + " if 0 <= nx-1 < 4 and new_board[nx-1][ny] == new_board[nx][ny] != 0:\n", + " new_board[nx-1][ny] *= 2\n", + " new_board[nx][ny] = 0\n", + " else:\n", + " if 0 <= ny-1 < 4 and new_board[nx][ny-1] == new_board[nx][ny] != 0:\n", + " new_board[nx][ny-1] *= 2\n", + " new_board[nx][ny] = 0\n", + " if not moved:\n", + " continue\n", + " score = sum(new_board[x][y] for x in range(4) for y in range(4))\n", + " if score > best_score:\n", + " best_score = score\n", + " best_move = move\n", + " return str(best_move)\n", + "┌───┬───┬───┬───┬───┬───┬───┐\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;33m 1\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;33m 1\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "└───┴───┴───┴───┴───┴───┴───┘\n", + "def strategy(board):\n", + " # simple heuristic: try to merge the first pair of equal tiles from left to right\n", + " for i in range(len(board)):\n", + " for j in range(len(board[i])-1):\n", + " if board[i][j] == board[i][j+1] and board[i][j] != 0:\n", + " return str(j) # direction: 0-left, 1-right, 2-up, 3-down\n", + " # if no merges, slide to fill empty spot on the left\n", + " for i in range(len(board)):\n", + " for j in range(len(board[i])):\n", + " if board[i][j] == 0:\n", + " return \"0\"\n", + " return \"0\"\n", + "Steps = 9 If Done = False\n", + "┌───┬───┬───┬───┬───┬───┬───┐\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;33m 1\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;33m 1\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "└───┴───┴───┴───┴───┴───┴───┘\n", + "Steps = 9 If Done = False\n", + "def strategy(board):\n", + " return \"0\"\n", + "┌───┬───┬───┬───┬───┬───┬───┐\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;33m 1\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;33m 1\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "└───┴───┴───┴───┴───┴───┴───┘\n", + "Steps = 9 If Done = False\n", + "def strategy(board):\n", + " # board is a list of 4 lists, each containing 4 integers (0 for empty)\n", + " from collections import Counter\n", + " # Count numbers of each value\n", + " counts = Counter([num for row in board for num in row if num != 0])\n", + " # Prefer moving toward the rightmost or downwards if a move will combine\n", + " # First, try to combine pairs by moving left\n", + " for i in range(4):\n", + " for j in range(1,4):\n", + " if board[i][j] == board[i][j-1] and board[i][j] != 0:\n", + " return \"3\" # Move up to combine\n", + " # If no direct combine, move right if possible\n", + " for i in range(4):\n", + " for j in range(3):\n", + " if board[i][j] == 0:\n", + " return \"2\" # Move right\n", + " # If no empty, move down\n", + " return \"1\"\n", + "┌───┬───┬───┬───┬───┬───┬───┐\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;33m 1\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;33m 1\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "└───┴───┴───┴───┴───┴───┴───┘\n", + "def strategy(board):\n", + " # Try to move left if possible, else right, up, down\n", + " def can(move):\n", + " for i,row in enumerate(board):\n", + " if move == \"0\" and i>0 and row[i-1]==0: return True\n", + " if move == \"1\" and i<3 and row[i+1]==0: return True\n", + " if move == \"2\" and i>0 and board[i-1][i]==0: return True\n", + " if move == \"3\" and i<3 and board[i+1][i]==0: return True\n", + " return False\n", + "\n", + " for m in [\"0\",\"1\",\"2\",\"3\"]:\n", + " if can(m):\n", + " return m\n", + " return \"0\"\n", + "Steps = 9 If Done = False\n", + "┌───┬───┬───┬───┬───┬───┬───┐\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;33m 1\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;33m 1\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "└───┴───┴───┴───┴───┴───┴───┘\n", + "Steps = 9 If Done = False\n", + "def strategy(board):\n", + " # Simple heuristic: move a tile towards the nearest zero cell.\n", + " n = len(board)\n", + " for i in range(n):\n", + " for j in range(n):\n", + " if board[i][j] != 0:\n", + " # try to move right if possible\n", + " if j+1 < n and board[i][j+1] == 0:\n", + " return \"1\" # move right\n", + " # try upwards\n", + " if i-1 >= 0 and board[i-1][j] == 0:\n", + " return \"0\" # move up\n", + " # try left\n", + " if j-1 >= 0 and board[i][j-1] == 0:\n", + " return \"3\" # move left\n", + " # try downwards\n", + " if i+1 < n and board[i+1][j] == 0:\n", + " return \"2\" # move down\n", + " return \"0\"\n", + "┌───┬───┬───┬───┬───┬───┬───┐\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;33m 1\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;33m 1\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "└───┴───┴───┴───┴───┴───┴───┘\n", + "Exception = list index out of range\n", + "None\n", + "Steps = 9 If Done = False\n", + "def strategy(board):\n", + " \"\"\"\n", + " Determine the best move (\"0\": Up, \"1\": Right, \"2\": Down, \"3\": Left)\n", + " for a 2048 board represented by a list of lists.\n", + " This implementation uses a simple heuristic: count the number of\n", + " empty cells after each potential move and choose the move that\n", + " maximizes this count. It does not simulate future moves.\n", + " \"\"\"\n", + " # Directions: 0=Up,1=Right,2=Down,3=Left\n", + " dirs = [( -1, 0), ( 0, 1), ( 1, 0), ( 0, -1)]\n", + " best_move = None\n", + " best_empty = -1\n", + "\n", + " n = len(board)\n", + " for move, (dx, dy) in enumerate(dirs):\n", + " new_board = [row[:] for row in board] # copy\n", + " changed = False\n", + " for i in range(n):\n", + " for j in range(n):\n", + " x, y = i, j\n", + " # Move the tile in the chosen direction\n", + " while True:\n", + " nx, ny = x + dx, y + dy\n", + " if 0 <= nx < n and 0 <= ny < n and new_board[nx][ny] == 0:\n", + " # Merge if possible\n", + " if new_board[x][y] != 0 and new_board[nx][ny] == 0:\n", + " new_board[nx][ny] = new_board[x][y]\n", + " new_board[x][y] = 0\n", + " changed = True\n", + " x, y = nx, ny\n", + " else:\n", + " break\n", + " # If tile could not move, skip\n", + " # Count empty cells in the resulting board\n", + " empty = sum(row.count(0) for row in new_board)\n", + " if empty > best_empty:\n", + " best_empty = empty\n", + " best_move = str(move)\n", + " return best_move\n", + "┌───┬───┬───┬───┬───┬───┬───┐\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;33m 1\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;33m 1\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "└───┴───┴───┴───┴───┴───┴───┘\n", + "Steps = 2 If Done = False\n", + "def strategy(board):\n", + " n = len(board)\n", + " score = lambda r,c: board[r][c]\n", + " # Count empty cells and total\n", + " empties = sum(board[i][j] == 0 for i in range(n) for j in range(n))\n", + " # Simple heuristic: move left if highest tile on left, else right, else up then down\n", + " # Find position of maximum tile\n", + " max_val = -1\n", + " max_pos = None\n", + " for i in range(n):\n", + " for j in range(n):\n", + " if board[i][j] > max_val:\n", + " max_val = board[i][j]\n", + " max_pos = (i, j)\n", + " # Prefer moving towards the edge with max tile\n", + " i, j = max_pos\n", + " # Prioritize directions that keep max tile towards edge\n", + " if j == 0: return \"0\" # left\n", + " if j == n-1: return \"1\" # right\n", + " if i == 0: return \"2\" # up\n", + " return \"3\" # down\n", + "┌───┬───┬───┬───┬───┬───┬───┐\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;33m 1\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "├───┼───┼───┼───┼───┼───┼───┤\n", + "│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;33m 1\u001b[0m│\u001b[38;5;239m .\u001b[0m│\u001b[38;5;239m .\u001b[0m│\n", + "└───┴───┴───┴───┴───┴───┴───┘\n" + ] + } + ], + "source": [ + "trainer.train()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "tlaUdxC_VHpz" + }, + "source": [ + "## Testing the Trained Model\n", + "\n", + "Let's generate a strategy from our RL-trained model and see how it differs from the base model:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "TwZygRdWf8ab" + }, + "outputs": [], + "source": [ + "text = tokenizer.apply_chat_template(\n", + " [{\"role\": \"user\", \"content\": prompt}],\n", + " tokenize=False,\n", + " add_generation_prompt=True,\n", + " reasoning_effort=\"low\",\n", + ")\n", + "\n", + "from transformers import TextStreamer\n", + "\n", + "_ = model.generate(\n", + " **tokenizer(text, return_tensors=\"pt\").to(\"cuda\"),\n", + " temperature=1.0,\n", + " max_new_tokens=1024,\n", + " streamer=TextStreamer(tokenizer, skip_prompt=False),\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "-NUEmHFSYNTp" + }, + "source": [ + "## Saving the Fine-tuned Model\n", + "\n", + "You can save the trained model in different formats:\n", + "\n", + "- **MXFP4**: OpenAI gpt-oss's native 4-bit precision format\n", + "- **float16**: Standard half-precision for broader compatibility\n", + "\n", + "To push to Hugging Face Hub, you'll need a token from https://huggingface.co/settings/tokens:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "NjXGTkp7YNtB" + }, + "outputs": [], + "source": [ + "# Merge and push to hub in mxfp4 4bit format\n", + "if False:\n", + " model.save_pretrained_merged(\"finetuned_model\", tokenizer, save_method=\"mxfp4\")\n", + "if False:\n", + " model.push_to_hub_merged(\"repo_id/repo_name\", tokenizer, token=\"hf...\", save_method=\"mxfp4\")\n", + "\n", + "# Merge and push to hub in 16bit\n", + "if False:\n", + " model.save_pretrained_merged(\"finetuned_model\", tokenizer, save_method=\"merged_16bit\")\n", + "if False: # Pushing to HF Hub\n", + " model.push_to_hub_merged(\"hf/gpt-oss-finetune\", tokenizer, save_method=\"merged_16bit\", token=\"\")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "V15Yhj1V9lwG" + }, + "source": [ + "## Conclusion\n", + "\n", + "Congratulations! You've learned how to apply reinforcement learning to teach an LLM to generate game-playing code. The key concepts covered:\n", + "\n", + "1. **OpenEnv** for standardized access to RL environments\n", + "2. **LoRA** for memory-efficient fine-tuning\n", + "3. **Sandboxed execution** to prevent reward hacking\n", + "4. **Multi-objective reward functions** that balance validity, safety, and performance\n", + "5. **GRPO** for policy optimization without a value network\n", + "\n", + "This pattern extends beyond 2048—you can adapt it to any task where model outputs can be programmatically evaluated: code synthesis, mathematical proofs, API usage, and more.\n", + "\n", + "### Further Resources\n", + "\n", + "- [OpenAI gpt-oss-20b Model Card](https://huggingface.co/openai/gpt-oss-20b)\n", + "- [OpenEnv Documentation](https://github.com/meta-pytorch/OpenEnv)\n", + "- [TRL GRPO Trainer](https://huggingface.co/docs/trl/main/en/grpo_trainer)\n", + "- [Unsloth RL Guide](https://docs.unsloth.ai/get-started/reinforcement-learning-rl-guide)\n", + "\n", + "---\n", + "\n", + "*This notebook uses [Unsloth](https://github.com/unslothai/unsloth) for memory-efficient training.*\n", + "\n", + "**License:** Apache 2.0" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "KMwNkyqlB4Ae" + }, + "source": [] + } + ], + "metadata": { + "accelerator": "GPU", + "colab": { + "gpuType": "T4", + "provenance": [] + }, + "kernelspec": { + "display_name": "openenv-crisisworldcortex", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.0" + }, + "widgets": { + "application/vnd.jupyter.widget-state+json": { + "004f28173c3b4fb6a9f8c2068f5db81f": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "02c88a690a384ae183c233b6927aaf57": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "050567dccb47456aaac65d118ac60a6b": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_ea2f3ca562444c46b50596a1d7cf9030", + "IPY_MODEL_0b7287d482cc44dbb406d71f23f1aea0", + "IPY_MODEL_474c7fe6ef4b430d9826171eded2ebf1" + ], + "layout": "IPY_MODEL_12a877304ddf45e49bbdfe056394c3d6" + } + }, + "06e420cfa2974f7d8d7ec4b83f064a6e": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "072060f8bdb54a15baf838f67d376d99": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_29d98af49dd3412f84f5843b937029d1", + "max": 3372033380, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_93176d6b63284b4e832e0f028be90655", + "value": 3372033380 + } + }, + "08d8f0cfd7614900a9c9bba888619749": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "0b7287d482cc44dbb406d71f23f1aea0": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_d541fd264d7e4b2691e8efbd993e6ae7", + "max": 446, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_cbc7bbfe983f4e6696f8dd6b37c50543", + "value": 446 + } + }, + "0b851acfd32047bfb6bae17d43ccfcb1": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_667192c07a7340a9a72ed25648a0be64", + "max": 3996690997, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_aa714b1f70e8495b9299b51d6ac4c3c4", + "value": 3996690997 + } + }, + "0b8fa4ff186a4bfeac18cf6d676e99df": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "0e49035e0c3a4ee4ab477b475e74ef36": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "0f09a00375bd4d4c8863fc8fb7d64d61": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "1055200185004ea2a95a05eb51232501": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "12a877304ddf45e49bbdfe056394c3d6": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "18d4cc3fd8ee4e3fbf27c574fd467f20": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "1b26eac03fed4c8784bd611474cf4607": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_b72fc22310714bf0bf6ff4021db5aba8", + "placeholder": "​", + "style": "IPY_MODEL_d2a7fa9dddc240e29330297870159c59", + "value": "Loading checkpoint shards: 100%" + } + }, + "1b84d7dc9567474c8587432d48342b71": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_665f7b7e5496432985ed9f49829f5834", + "placeholder": "​", + "style": "IPY_MODEL_d16dc921c6554bc6b77abcb423721cc8", + "value": " 1.19M/? [00:00<00:00, 76.2MB/s]" + } + }, + "1daa373c890b4ae0a9cf6a3ec325693c": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "1ece5fe9597a4e3db2dd96c38995705c": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "212c17e3829c4accb30265a3d9ee73dc": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "2612008cf36949d9a6618a01ac817618": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "28300c16023d4ad9a59784baea2f57aa": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": "20px" + } + }, + "297bdff1add5414893319b185cb15da6": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_362947c7e102474cbf560f51e713bdff", + "IPY_MODEL_7f63ff3e87aa4b86a4f8e64785d1d34c", + "IPY_MODEL_cca7d922b85449a1b0c5c025b65fba10" + ], + "layout": "IPY_MODEL_5520c59f24cd414092bf5f952425611d" + } + }, + "29d98af49dd3412f84f5843b937029d1": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "2d91f25a8d0b4dd39dc320b2cf17fc0b": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_88aa58551344410a91b07af480d6ab53", + "max": 1, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_4f38070c09354406928c5e7be6cff3fd", + "value": 1 + } + }, + "2f57c8e713b94c8692837b4e17c9e983": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "31afdb187d2a44d8b3a101fa18543c13": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "3371a1da5d0e426bb6cc02a6c383dd6f": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "3566e9058ebc45498b42e521f2314365": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "362947c7e102474cbf560f51e713bdff": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_b1dcd2f908344949af01dab5a244e458", + "placeholder": "​", + "style": "IPY_MODEL_9e6f46ddb61943f4a168d70a209a7ddc", + "value": "model-00004-of-00004.safetensors: 100%" + } + }, + "383e9b4b74e34cae96c1f46f41591b82": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "3c2e00b9d20a4c9ea5b850b776752fd2": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_7c29e5f727ec4b608c06d0487a2c9e53", + "IPY_MODEL_4beae5415d0647e2898a83313a08ea94", + "IPY_MODEL_62cd4ccc430549e7a2c156222d47ebeb" + ], + "layout": "IPY_MODEL_4691273248984fdea29d83ec0a246cd9" + } + }, + "3e3ce50c437a412f9cc2bedb697648f7": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "3f589cf3cb804ed29ba322e4fa10c511": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "419e4369b36644d3abec159511a88ad8": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_eee0d2b6eb6047d8a4001f4999dcdc35", + "IPY_MODEL_2d91f25a8d0b4dd39dc320b2cf17fc0b", + "IPY_MODEL_1b84d7dc9567474c8587432d48342b71" + ], + "layout": "IPY_MODEL_f34cadfaf9bb46729aeeff9492ba9026" + } + }, + "41ee8407344d402997b0574e0ae26c77": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "4691273248984fdea29d83ec0a246cd9": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "46cb3593b37c4cb9b4bac422bc5809c8": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "474c7fe6ef4b430d9826171eded2ebf1": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_f290b17e3cd9487b9b97d54d6cec9efc", + "placeholder": "​", + "style": "IPY_MODEL_18d4cc3fd8ee4e3fbf27c574fd467f20", + "value": " 446/446 [00:00<00:00, 50.9kB/s]" + } + }, + "48f65e25acd84b0cb582de66753215da": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_383e9b4b74e34cae96c1f46f41591b82", + "max": 165, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_46cb3593b37c4cb9b4bac422bc5809c8", + "value": 165 + } + }, + "4ba91b8d008e483d89f16f204326f6b6": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "4bc16bd2399a43fe85967131af7f846a": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "4beae5415d0647e2898a83313a08ea94": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_a6a122064bc340868b5e0e11afa9c42f", + "max": 1, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_3371a1da5d0e426bb6cc02a6c383dd6f", + "value": 1 + } + }, + "4ca56d8605864a438290152268bfc686": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_2612008cf36949d9a6618a01ac817618", + "max": 3998751275, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_3e3ce50c437a412f9cc2bedb697648f7", + "value": 3998751275 + } + }, + "4dc823fcd0dd4eaaa2e8aaff0daa9ad1": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_1b26eac03fed4c8784bd611474cf4607", + "IPY_MODEL_a24e2cfbef794d35a0e22753352caa15", + "IPY_MODEL_8368e4420e814c6f9be30994b69c66ee" + ], + "layout": "IPY_MODEL_eb8c197ce1fc41f78531e5e73ae14a89" + } + }, + "4e63263fc27b4f07b4f6abffba082379": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_eb13ad96565a44519e7cab9ce9483b90", + "IPY_MODEL_0b851acfd32047bfb6bae17d43ccfcb1", + "IPY_MODEL_d4c72002b5fe44d3ad7ae67f5536c889" + ], + "layout": "IPY_MODEL_a1e042e5b8ad4b028cc28ac54924207d" + } + }, + "4f29591af0d64d398515479b032a1b3d": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "4f38070c09354406928c5e7be6cff3fd": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "52e9a0ea76df4fd8822ccadadaadb501": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "5371b2fad7b04f97bd8f4671d844d2cb": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_9c2371c32afe46519ee53427ea42bc9a", + "placeholder": "​", + "style": "IPY_MODEL_f9d4672fb86b4b4e9c3e9068dc479e5b", + "value": " 3.37G/3.37G [00:20<00:00, 60.9MB/s]" + } + }, + "5483233a9b224f3c8eb9337e4ed82314": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "5520c59f24cd414092bf5f952425611d": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "56b30cc150924879abfc138427f4ca98": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "5d1e2fdbf7a2409abbafb63e4b160668": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_a4c19dc98fe943e09a26a60d23c8ff01", + "IPY_MODEL_8f5799610318490492ff5aba76be3d1a", + "IPY_MODEL_e78534b135d2465c83e3be614b71c8a4" + ], + "layout": "IPY_MODEL_2f57c8e713b94c8692837b4e17c9e983" + } + }, + "62cd4ccc430549e7a2c156222d47ebeb": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_bedc42e908454f1494768194b43b2964", + "placeholder": "​", + "style": "IPY_MODEL_41ee8407344d402997b0574e0ae26c77", + "value": " 22.8k/? [00:00<00:00, 1.59MB/s]" + } + }, + "665f7b7e5496432985ed9f49829f5834": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "667192c07a7340a9a72ed25648a0be64": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "6af6de3285684e76b320571b44af5fc1": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_f7cc1614e13d4d22b83f787b20407a5f", + "IPY_MODEL_072060f8bdb54a15baf838f67d376d99", + "IPY_MODEL_5371b2fad7b04f97bd8f4671d844d2cb" + ], + "layout": "IPY_MODEL_b10d25fb43eb42198abf71c4e326bcff" + } + }, + "6d77c8dcb28240f7b860571d11f8b9af": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "754f2452fbe14c7098215ec810ffbf14": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "7c29e5f727ec4b608c06d0487a2c9e53": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_ae3819cd042f43babef24199081bb97f", + "placeholder": "​", + "style": "IPY_MODEL_52e9a0ea76df4fd8822ccadadaadb501", + "value": "tokenizer_config.json: " + } + }, + "7c52841c7e714173bfd526b0a625bc9d": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_ae9728c04b974af29460ce5179a9edba", + "placeholder": "​", + "style": "IPY_MODEL_754f2452fbe14c7098215ec810ffbf14", + "value": " 165/165 [00:00<00:00, 16.4kB/s]" + } + }, + "7c7d40163ecc4dae8a2c54af21de4661": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "7ce0f239a8514923b383f738bc0c9899": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "7f63ff3e87aa4b86a4f8e64785d1d34c": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_ae79b9a378ee4218beddf77ac9af6de7", + "max": 1158267008, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_e6d63cf58647443b9749195ec0579d87", + "value": 1158267008 + } + }, + "8368e4420e814c6f9be30994b69c66ee": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_d1cea390ecaa4583a698278fa3a438c9", + "placeholder": "​", + "style": "IPY_MODEL_5483233a9b224f3c8eb9337e4ed82314", + "value": " 4/4 [00:56<00:00, 12.02s/it]" + } + }, + "8708226010324a83b4f7900c3958d430": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_4ba91b8d008e483d89f16f204326f6b6", + "placeholder": "​", + "style": "IPY_MODEL_31afdb187d2a44d8b3a101fa18543c13", + "value": "chat_template.jinja: " + } + }, + "88aa58551344410a91b07af480d6ab53": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": "20px" + } + }, + "89e3ad0d517f44d084df0d8a3ed40703": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "8f5799610318490492ff5aba76be3d1a": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_1055200185004ea2a95a05eb51232501", + "max": 27868174, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_eb1068efb4364ee2893c4d21c58f38db", + "value": 27868174 + } + }, + "93176d6b63284b4e832e0f028be90655": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "941d8cbf8188402eb603c18b6e979035": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_8708226010324a83b4f7900c3958d430", + "IPY_MODEL_fc12c97d659640b3b2b2b48ad7f17e5b", + "IPY_MODEL_aa22cb2d1dcf4001bd3720b075318906" + ], + "layout": "IPY_MODEL_bb358c5523814376a2d4690f88f20b74" + } + }, + "9c2371c32afe46519ee53427ea42bc9a": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "9e6f46ddb61943f4a168d70a209a7ddc": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "a1e042e5b8ad4b028cc28ac54924207d": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "a24e2cfbef794d35a0e22753352caa15": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_0e49035e0c3a4ee4ab477b475e74ef36", + "max": 4, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_08d8f0cfd7614900a9c9bba888619749", + "value": 4 + } + }, + "a4c19dc98fe943e09a26a60d23c8ff01": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_f8d2164046fb46a298e2d80628808cb3", + "placeholder": "​", + "style": "IPY_MODEL_af23432e17664cd6852496e43f9de0cf", + "value": "tokenizer.json: 100%" + } + }, + "a6a122064bc340868b5e0e11afa9c42f": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": "20px" + } + }, + "aa22cb2d1dcf4001bd3720b075318906": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_0b8fa4ff186a4bfeac18cf6d676e99df", + "placeholder": "​", + "style": "IPY_MODEL_ccb581e230604bb690015eb685e4b8e1", + "value": " 15.1k/? [00:00<00:00, 1.44MB/s]" + } + }, + "aa714b1f70e8495b9299b51d6ac4c3c4": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "ae3819cd042f43babef24199081bb97f": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "ae79b9a378ee4218beddf77ac9af6de7": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "ae9728c04b974af29460ce5179a9edba": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "af23432e17664cd6852496e43f9de0cf": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "b09745899e1446129f397d822a21fc99": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "b10d25fb43eb42198abf71c4e326bcff": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "b1dcd2f908344949af01dab5a244e458": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "b72fc22310714bf0bf6ff4021db5aba8": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "baf3db00d28f4c849bb6a6739e908c62": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_f4f2fb240a12406ab5e709be33b93683", + "IPY_MODEL_4ca56d8605864a438290152268bfc686", + "IPY_MODEL_de306a50594f463ba9c78c713bc33241" + ], + "layout": "IPY_MODEL_0f09a00375bd4d4c8863fc8fb7d64d61" + } + }, + "bb358c5523814376a2d4690f88f20b74": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "bd8f575034c041be93ff20f63388de2d": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "bedc42e908454f1494768194b43b2964": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "c4d592366499414a99f19bce7f0bd665": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "cbc7bbfe983f4e6696f8dd6b37c50543": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "cbed0dab2d7540f697eacec5a33e1061": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_dc9b2549ff834880ac19b578adfee5a5", + "IPY_MODEL_48f65e25acd84b0cb582de66753215da", + "IPY_MODEL_7c52841c7e714173bfd526b0a625bc9d" + ], + "layout": "IPY_MODEL_bd8f575034c041be93ff20f63388de2d" + } + }, + "cca7d922b85449a1b0c5c025b65fba10": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_e7ca6b7ba2094872a888da5511e2bb49", + "placeholder": "​", + "style": "IPY_MODEL_7c7d40163ecc4dae8a2c54af21de4661", + "value": " 1.16G/1.16G [00:09<00:00, 242MB/s]" + } + }, + "ccb581e230604bb690015eb685e4b8e1": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "d16dc921c6554bc6b77abcb423721cc8": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "d1cea390ecaa4583a698278fa3a438c9": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "d257eaa588bd41fb947f81d306fe05cd": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "d2a7fa9dddc240e29330297870159c59": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "d4c72002b5fe44d3ad7ae67f5536c889": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_06e420cfa2974f7d8d7ec4b83f064a6e", + "placeholder": "​", + "style": "IPY_MODEL_3566e9058ebc45498b42e521f2314365", + "value": " 4.00G/4.00G [00:19<00:00, 279MB/s]" + } + }, + "d541fd264d7e4b2691e8efbd993e6ae7": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "d82ab09313cc482f9b9b45192f489825": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "dc9b2549ff834880ac19b578adfee5a5": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_212c17e3829c4accb30265a3d9ee73dc", + "placeholder": "​", + "style": "IPY_MODEL_1daa373c890b4ae0a9cf6a3ec325693c", + "value": "generation_config.json: 100%" + } + }, + "de306a50594f463ba9c78c713bc33241": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_fa5c42218fbb44378bef71551c3383e0", + "placeholder": "​", + "style": "IPY_MODEL_d82ab09313cc482f9b9b45192f489825", + "value": " 4.00G/4.00G [00:25<00:00, 110MB/s]" + } + }, + "e6d63cf58647443b9749195ec0579d87": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "e78534b135d2465c83e3be614b71c8a4": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_c4d592366499414a99f19bce7f0bd665", + "placeholder": "​", + "style": "IPY_MODEL_1ece5fe9597a4e3db2dd96c38995705c", + "value": " 27.9M/27.9M [00:01<00:00, 21.9MB/s]" + } + }, + "e7ca6b7ba2094872a888da5511e2bb49": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "ea2f3ca562444c46b50596a1d7cf9030": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_56b30cc150924879abfc138427f4ca98", + "placeholder": "​", + "style": "IPY_MODEL_02c88a690a384ae183c233b6927aaf57", + "value": "special_tokens_map.json: 100%" + } + }, + "eb1068efb4364ee2893c4d21c58f38db": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "eb13ad96565a44519e7cab9ce9483b90": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_6d77c8dcb28240f7b860571d11f8b9af", + "placeholder": "​", + "style": "IPY_MODEL_4f29591af0d64d398515479b032a1b3d", + "value": "model-00002-of-00004.safetensors: 100%" + } + }, + "eb8c197ce1fc41f78531e5e73ae14a89": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "eee0d2b6eb6047d8a4001f4999dcdc35": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_d257eaa588bd41fb947f81d306fe05cd", + "placeholder": "​", + "style": "IPY_MODEL_b09745899e1446129f397d822a21fc99", + "value": "model.safetensors.index.json: " + } + }, + "f290b17e3cd9487b9b97d54d6cec9efc": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "f34cadfaf9bb46729aeeff9492ba9026": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "f4f2fb240a12406ab5e709be33b93683": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_89e3ad0d517f44d084df0d8a3ed40703", + "placeholder": "​", + "style": "IPY_MODEL_3f589cf3cb804ed29ba322e4fa10c511", + "value": "model-00001-of-00004.safetensors: 100%" + } + }, + "f7cc1614e13d4d22b83f787b20407a5f": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_7ce0f239a8514923b383f738bc0c9899", + "placeholder": "​", + "style": "IPY_MODEL_4bc16bd2399a43fe85967131af7f846a", + "value": "model-00003-of-00004.safetensors: 100%" + } + }, + "f8d2164046fb46a298e2d80628808cb3": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "f9d4672fb86b4b4e9c3e9068dc479e5b": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "fa5c42218fbb44378bef71551c3383e0": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "fc12c97d659640b3b2b2b48ad7f17e5b": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_28300c16023d4ad9a59784baea2f57aa", + "max": 1, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_004f28173c3b4fb6a9f8c2068f5db81f", + "value": 1 + } + } + } + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/uv.lock b/uv.lock index fe57598451743b6183eaf07bdb2c03f239205dd4..b10b86d225278c961188eb7c72d020c8df6a3019 100644 --- a/uv.lock +++ b/uv.lock @@ -1650,6 +1650,7 @@ source = { editable = "." } dependencies = [ { name = "openai" }, { name = "openenv-core", extra = ["core"] }, + { name = "python-dotenv" }, ] [package.optional-dependencies] @@ -1667,6 +1668,7 @@ requires-dist = [ { name = "pre-commit", marker = "extra == 'dev'", specifier = ">=4.0.0" }, { name = "pytest", marker = "extra == 'dev'", specifier = ">=8.0.0" }, { name = "pytest-cov", marker = "extra == 'dev'", specifier = ">=4.0.0" }, + { name = "python-dotenv", specifier = ">=1.0.0" }, { name = "ruff", marker = "extra == 'dev'", specifier = ">=0.8.0" }, ] provides-extras = ["dev"]