src/rune_goblin/evaluate.py

"""Evaluate the fine-tuned model like a game engine (section 14).

Metrics: valid-JSON rate, rune-rule accuracy (does a weakness-hitting spell
deal extra damage?), enemy-weakness usage, damage-range validity, and a
crude spell-name diversity score.

Usage::

    uv run python -m rune_goblin.evaluate --n 200
"""

from __future__ import annotations

import argparse
import random

from .engine import GameState, resolve_spell
from .inference import get_model
from .runelang import ALL_ENEMIES, ALL_RUNES, ENEMIES, rune_matches


def _random_case(rng: random.Random) -> tuple[GameState, list[str]]:
    enemy = rng.choice(ALL_ENEMIES)
    e = ENEMIES[enemy]
    state = GameState(
        player_hp=rng.randint(3, 10), enemy_name=enemy,
        enemy_hp=e.max_hp, enemy_max_hp=e.max_hp, room_mood=e.mood,
    )
    runes = rng.sample(ALL_RUNES, k=rng.randint(2, 4))
    return state, runes


def main() -> None:
    ap = argparse.ArgumentParser()
    ap.add_argument("--n", type=int, default=200)
    ap.add_argument("--seed", type=int, default=123)
    ap.add_argument("--rules-only", action="store_true", help="evaluate the rule engine, not the model")
    args = ap.parse_args()

    rng = random.Random(args.seed)
    model = None if args.rules_only else get_model()
    using = "rule-engine" if model is None else "fine-tuned model"
    print(f"Evaluating {using} on {args.n} cases\n")

    valid = 0
    weakness_used = 0
    weakness_cases = 0
    damage_ok = 0
    names: set[str] = set()

    for _ in range(args.n):
        state, runes = _random_case(rng)
        if model is not None:
            spell = model.cast(state, runes)
            raw_valid = spell is not None
        else:
            spell = resolve_spell(state, runes, seed=rng.randint(0, 2**31 - 1))
            raw_valid = True

        if not raw_valid or spell is None:
            continue
        valid += 1
        names.add(spell.spell_name)

        # damage range: enemy_hp_delta must be within [-enemy_hp, 0..]
        if -state.enemy_hp <= spell.enemy_hp_delta <= state.enemy_max_hp:
            damage_ok += 1

        hits_weak = any(rune_matches(r, state.weakness) for r in runes)
        if hits_weak:
            weakness_cases += 1
            # heuristic: hitting a weakness should produce meaningful damage
            if spell.enemy_hp_delta <= -2 or "weakness_revealed" in spell.status_effects:
                weakness_used += 1

    n = args.n
    print(f"Valid JSON rate     : {valid / n:.1%}  (goal >95%)")
    print(f"Damage range valid  : {damage_ok / max(1, valid):.1%}  (goal >95%)")
    if weakness_cases:
        print(f"Weakness usage      : {weakness_used / weakness_cases:.1%}  (goal >80%)")
    print(f"Spell name diversity: {len(names)}/{valid} unique")


if __name__ == "__main__":
    main()