File size: 3,985 Bytes
ddb2889 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 | from __future__ import annotations
import argparse
import json
import sys
import time
from dataclasses import replace
from pathlib import Path
from typing import Any
import numpy as np
ROOT = Path(__file__).resolve().parents[1]
if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
from life_game.arcade import (
ArcadeConfig,
ChallengeSpecError,
LLMChallengeError,
apply_challenge_spec,
parse_challenge_spec,
request_llm_challenge,
)
from life_game.board import BoardModel
from life_game.game import INTERVENTION_MODE, TOWER_MODE, new_game
def main() -> int:
parser = argparse.ArgumentParser(description="Probe a challenge-spec LLM endpoint.")
parser.add_argument("--base-url", default="http://127.0.0.1:8080/v1")
parser.add_argument("--model", default="signal-garden-qwen35-code-mutation")
parser.add_argument("--timeout-seconds", type=float, default=120.0)
parser.add_argument("--fail-on-error", action="store_true")
args = parser.parse_args()
config = ArcadeConfig(
llm_base_url=args.base_url,
llm_model=args.model,
llm_timeout_seconds=max(1.0, args.timeout_seconds),
)
rng = np.random.default_rng(3407)
cases = [
{
"name": "intervention_start",
"mode": INTERVENTION_MODE,
"preference": "readable survival pressure",
"phase": "start",
},
{
"name": "intervention_mutation",
"mode": INTERVENTION_MODE,
"preference": "readable survival pressure",
"phase": "mutation",
"score": 42,
"data": {"discharges": 1},
},
{
"name": "tower_start",
"mode": TOWER_MODE,
"preference": "tower defense with light pressure",
"phase": "start",
},
]
results = []
for case in cases:
result = _run_case(config, case, rng)
results.append(result)
print(json.dumps(result, sort_keys=True))
passed = sum(1 for result in results if result["ok"])
summary = {"ok": passed == len(results), "passed": passed, "total": len(results)}
print(json.dumps({"summary": summary}, sort_keys=True))
return 1 if args.fail_on_error and not summary["ok"] else 0
def _run_case(config: ArcadeConfig, case: dict[str, Any], rng: np.random.Generator) -> dict[str, Any]:
model = BoardModel(grid=np.zeros((16, 16), dtype=np.uint8))
model = replace(model, running=True)
game = new_game(model.size, str(case["mode"]), rng)
if "score" in case:
game = replace(game, score=int(case["score"]))
if "data" in case:
game = replace(game, data={**game.data, **case["data"]})
started = time.monotonic()
try:
raw = request_llm_challenge(
config,
str(case["mode"]),
str(case["preference"]),
case["phase"],
model,
game,
)
spec = parse_challenge_spec(raw, case["phase"])
next_model, next_game = apply_challenge_spec(model, game, spec, case["phase"], rng)
except (ChallengeSpecError, LLMChallengeError) as exc:
return {
"name": case["name"],
"ok": False,
"phase": case["phase"],
"error": str(exc),
"elapsed_seconds": round(time.monotonic() - started, 3),
}
return {
"name": case["name"],
"ok": True,
"phase": case["phase"],
"title": spec.title,
"difficulty": spec.difficulty,
"actions": [action.type for action in spec.actions],
"action_count": len(spec.actions),
"goal_updates": dict(spec.goal_updates),
"enemy_delta": next_game.enemy_count - game.enemy_count,
"live_cell_delta": int(next_model.grid.sum()) - int(model.grid.sum()),
"elapsed_seconds": round(time.monotonic() - started, 3),
}
if __name__ == "__main__":
raise SystemExit(main())
|