eval_felt_quality.py

"""
eval_felt_quality.py — eyeball the model's affect judgments on a fixed, varied
sentence set, and render each as its shape+color. Use it to (a) sanity-check that
judgments feel right, and (b) A/B models (Qwen3 8B vs MiniCPM4.1-8B vs ...).

Run (model comes from the same env vars as the app):
    STORY_SHAPES_MODEL=qwen3:8b            python eval_felt_quality.py
    STORY_SHAPES_MODEL=openbmb/MiniCPM4.1-8B python eval_felt_quality.py   # if pulled in Ollama

Outputs:
    eval_<model>.png   a grid: sentence -> judged V/A/D -> rendered shape+color
    eval_<model>.json  raw judgments for diffing between models

The sentence set spans the space ON PURPOSE: calm/sad/angry/joyful, literal vs
poetic vs absurd, understated vs loaded — so sameness or mis-judgment shows up.
"""
import os, json, sys

SENTENCES = [
    # calm / pleasant
    "The lake lay perfectly still under the morning light.",
    "She sipped her tea and watched the snow fall.",
    # sad / low
    "The last letter sat unopened on the empty table.",
    "He walked home alone in the grey rain.",
    # angry / threatening
    "The blade flashed once and the room went silent.",
    "Glass shattered as the door slammed off its hinges.",
    # joyful / high-valence high-arousal
    "Children spilled into the square, laughing and shrieking with delight.",
    "Fireworks burst gold across the whole sky at once.",
    # tense / suspense (mid arousal, low valence)
    "Something shifted in the dark just beyond the candlelight.",
    # commanding / high dominance
    "The mountain loomed over the valley, vast and unmoving.",
    # timid / low dominance
    "A small mouse trembled at the edge of the floorboard.",
    # absurd / nonsensical (judge by feeling, not sense)
    "A flying rhinoceros with fire wings sneezes Tuesday.",
    "Purple seven runs sideways into the soft idea of soup.",
    # poetic / understated
    "Dusk folded itself quietly over the rooftops.",
    # loaded / over-stuffed
    "Rage and grief and joy and terror crashed through her all at once.",
]

def main():
    from model import backend
    model = backend.MODEL
    safe = model.replace("/", "_").replace(":", "_")
    results = []
    print(f"Evaluating model: {model}\n")
    for s in SENTENCES:
        try:
            j = backend.judge_beat(s, story="", mode="exploration")
            v, a, d = j["valence"], j["arousal"], j["dominance"]
            print(f"  V{v:.2f} A{a:.2f} D{d:.2f}  deserve={str(j.get('deserves_shape')):5s}  {s[:55]}")
            results.append({"sentence": s, **j})
        except Exception as e:
            print(f"  [FAILED] {s[:55]}  ({e})")
            results.append({"sentence": s, "error": str(e)})

    with open(f"eval_{safe}.json", "w") as f:
        json.dump({"model": model, "results": results}, f, indent=2)
    print(f"\nwrote eval_{safe}.json")

    # render the grid (needs matplotlib + the engine; safe to skip if headless)
    try:
        import matplotlib; matplotlib.use("Agg")
        import matplotlib.pyplot as plt
        from engine.mappings import affect_to_geometry, affect_to_color
        from engine.renderer import geom_to_points
        ok = [r for r in results if "valence" in r]
        n = len(ok); cols = 4; rows = (n + cols - 1) // cols
        fig, axes = plt.subplots(rows, cols, figsize=(cols * 3, rows * 3.2))
        axes = axes.ravel()
        for ax, r in zip(axes, ok):
            g = affect_to_geometry(r["valence"], r["arousal"], r["dominance"])
            col = affect_to_color(r["valence"], r["arousal"], r["dominance"])
            pts = geom_to_points(g, seed_key=r["sentence"])
            ax.add_patch(plt.Polygon(pts, closed=True, facecolor=col, edgecolor="#222", lw=1.2))
            ax.set_xlim(0, 1); ax.set_ylim(1, 0); ax.set_aspect("equal"); ax.axis("off")
            ax.set_title(r["sentence"][:38] + ("…" if len(r["sentence"]) > 38 else ""), fontsize=7)
            ax.text(0.5, -0.04, f"V{r['valence']:.2f} A{r['arousal']:.2f} D{r['dominance']:.2f}",
                    fontsize=6.5, ha="center", va="top", transform=ax.transAxes, color="#777")
        for ax in axes[n:]:
            ax.axis("off")
        plt.suptitle(f"Felt-quality eval — {model}", fontsize=11, y=1.0)
        plt.tight_layout()
        plt.savefig(f"eval_{safe}.png", dpi=90, bbox_inches="tight")
        print(f"wrote eval_{safe}.png")
    except Exception as e:
        print(f"(skipped grid render: {e})")

if __name__ == "__main__":
    main()