scripts/run_mixed.py

"""Run the aligne panel on the 165-item mixed concept+emoji set, for both
base Gemma-3-27B-it and the functional-wellbeing FT. The mixed set is meant
to elicit emoji preferences through the still-working concept A/B channel,
sidestepping the slot-bias collapse seen in the emoji-only panel.

Run on the pod:
    uv run python scripts/run_mixed.py
"""

from __future__ import annotations

import asyncio
import sys
from dataclasses import asdict
from pathlib import Path

sys.path.insert(0, str(Path(__file__).resolve().parent))
from _logging import make_run_dir  # noqa: E402

from aligne.client import ChatClient, Endpoint  # noqa: E402
from aligne.metrics.preferences import PanelConfig, run_panel  # noqa: E402

REPO = Path(__file__).resolve().parent.parent
MIXED = REPO / "data" / "mixed_concepts_emoji.json"


async def one(name: str, model: str) -> None:
    cfg = PanelConfig(n_concepts=165, seed=0, concepts_path=MIXED)
    run_dir = make_run_dir(
        name,
        config={
            "name": name,
            "model": model,
            "concepts_path": str(MIXED),
            "panel_cfg": asdict(cfg),
        },
    )
    out_dir = run_dir / "aligne"
    out_dir.mkdir(parents=True, exist_ok=True)
    endpoint = Endpoint(base_url="http://localhost:8000/v1", model=model, api_key="dummy")
    client = ChatClient(endpoint=endpoint, concurrency=32, cache_path=run_dir / "client_cache.sqlite")
    print(f"[{name}] running mixed panel with model={model}", flush=True)
    panel = await run_panel(client, cfg, out_dir)
    print(
        f"[{name}] decisiveness={panel['decisiveness']:.4f} "
        f"unidim_r2={panel['unidim_r2']:.4f} n_edges={panel['n_edges']} "
        f"n_unanswered={panel['n_unanswered']} position_bias={panel['position_bias']:+.4f}",
        flush=True,
    )


async def main() -> None:
    await one("base_mixed", "google/gemma-3-27b-it")
    await one("ft_mixed", "functional-wellbeing")


if __name__ == "__main__":
    asyncio.run(main())