scripts/run_llm.py

"""Run the wood on the real small model via the deployed Modal endpoint.

Doubles as the 3B reliability gate: it reports how often the model emits a
parseable JSON object and at least one valid offer. If 3B holds up, we keep
Tiny Titan eligibility; if not, fall back to 7B (set TTW_MODEL and redeploy).

Run (after `python -m modal deploy serve.py`):
    python scripts/run_llm.py
"""

import sys
from pathlib import Path

sys.path.insert(0, str(Path(__file__).resolve().parents[1]))

from ttw.actions import parse_actions
from ttw.agents import make_llm_policy
from ttw.events import EventDeck
from ttw.llm import ModalLLM
from ttw.market import gini
from ttw.sim import step
from ttw.world import seed_world


def main(turns: int = 15) -> None:
    world = seed_world()
    client = ModalLLM()
    policy = make_llm_policy(client, temperature=0.7)
    deck = EventDeck(seed=7)

    parsed_ok = 0
    json_ok = 0
    total = 0

    for _ in range(turns):
        # Fate draws a Wood Legend (a reskinned market mania) every 5 turns.
        if world.turn in (4, 9, 13):
            legend = deck.draw(world)
            print(f"\n*** WOOD LEGEND: {legend.title} ***")
            print(f"    {legend.flavor}  (inspired by: {legend.inspired_by})")

        events = step(world, policy)
        raw = policy.state["raw"]

        # Reliability accounting for this turn.
        for name, text in raw.items():
            total += 1
            offers, _ = parse_actions(name, text)
            from ttw.actions import _extract_json

            if _extract_json(text) is not None:
                json_ok += 1
            if offers:
                parsed_ok += 1

        trades = [e for e in events if e["type"] == "trade"]
        gossip = [e for e in events if e["type"] == "gossip"]
        prices = " ".join(f"{g}={p:.0f}" for g, p in world.last_price.items())
        nets = [c.net_worth(world.last_price) for c in world.alive()]
        print(f"\n=== turn {world.turn} | {len(trades)} trades | gini {gini(nets):.2f} | {prices}")
        for t in trades:
            print(f"   {t['buyer']} bought {t['qty']} {t['good']} from {t['seller']} @ {t['price']}")
        for g in gossip:
            print(f"   [rumor] {g['creature']}: {g['message']}")

    # One sample reasoning trace, to eyeball quality.
    sample_name = next(iter(policy.state["raw"]))
    print(f"\n--- sample reasoning trace ({sample_name}, last turn) ---")
    print(policy.state["raw"][sample_name].strip()[:500])

    print("\n=== RELIABILITY ===")
    print(f"valid JSON object:      {json_ok}/{total} ({100*json_ok/total:.0f}%)")
    print(f"yielded >=1 valid offer: {parsed_ok}/{total} ({100*parsed_ok/total:.0f}%)")
    print("\nFinal standings:")
    for c in sorted(world.alive(), key=lambda x: -x.net_worth(world.last_price)):
        print(f"  {c.name:<9} pebbles={c.pebbles:>4} wellbeing={c.wellbeing} net={c.net_worth(world.last_price):.0f}")


if __name__ == "__main__":
    main()