backend/tests/test_compare.py

"""API tests for POST /api/compare.

Unit tests (SKIP_MODEL_LOAD=1) — no torch. The compare endpoint reads models
from app.state.model_cache via get_or_load_model, so client_with_compare_models
pre-seeds that cache with fakes: a 3-class FakeModel for twitter-roberta and a
2-class FakeBinaryModel for distilbert-sst2. Behavior is checked over HTTP only.
"""


def test_compare_defaults_to_two_models(client_with_compare_models):
    resp = client_with_compare_models.post("/api/compare", json={"text": "I love this"})
    assert resp.status_code == 200
    results = resp.json()["results"]
    assert [r["model_id"] for r in results] == ["twitter-roberta", "distilbert-sst2"]


def test_binary_model_returns_only_its_own_label_keys(client_with_compare_models):
    resp = client_with_compare_models.post("/api/compare", json={"text": "meh"})
    by_id = {r["model_id"]: r for r in resp.json()["results"]}
    # Binary DistilBERT has no neutral class — its scores must not fake one.
    assert set(by_id["distilbert-sst2"]["scores"]) == {"negative", "positive"}
    # The 3-class model keeps all three keys.
    assert set(by_id["twitter-roberta"]["scores"]) == {"negative", "neutral", "positive"}


def test_confidence_equals_max_score(client_with_compare_models):
    resp = client_with_compare_models.post("/api/compare", json={"text": "good"})
    for r in resp.json()["results"]:
        assert r["confidence"] == max(r["scores"].values())


def test_latency_is_numeric_and_non_negative(client_with_compare_models):
    resp = client_with_compare_models.post("/api/compare", json={"text": "good"})
    for r in resp.json()["results"]:
        assert isinstance(r["latency_ms"], (int, float))
        assert r["latency_ms"] >= 0


def test_explicit_model_ids_are_honored(client_with_compare_models):
    resp = client_with_compare_models.post(
        "/api/compare", json={"text": "good", "model_ids": ["distilbert-sst2"]}
    )
    results = resp.json()["results"]
    assert [r["model_id"] for r in results] == ["distilbert-sst2"]


def test_row_carries_registry_metadata(client_with_compare_models):
    resp = client_with_compare_models.post(
        "/api/compare", json={"text": "good", "model_ids": ["twitter-roberta"]}
    )
    row = resp.json()["results"][0]
    # name/domain/note come straight from the registry ModelConfig.
    assert row["name"] == "cardiffnlp/twitter-roberta-base-sentiment-latest"
    assert row["domain"] == "social / short English text"
    assert row["note"]


def test_rejects_non_sentiment_model_id(client_with_compare_models):
    resp = client_with_compare_models.post(
        "/api/compare", json={"text": "good", "model_ids": ["desklib-ai-detector"]}
    )
    assert resp.status_code == 400
    assert "sentiment" in resp.json()["detail"].lower()


def test_rejects_unknown_model_id(client_with_compare_models):
    resp = client_with_compare_models.post(
        "/api/compare", json={"text": "good", "model_ids": ["not-a-real-model"]}
    )
    assert resp.status_code == 400
    assert "not-a-real-model" in resp.json()["detail"]


def test_compare_rejects_disabled_model(monkeypatch, client_with_model):
    # ENABLED_MODELS is the public-deployment allowlist (Task 16A): the free
    # Space must not let anonymous users lazy-load every registry model. The
    # guard runs BEFORE get_or_load_model, so no fake cache entry is needed.
    monkeypatch.setenv("ENABLED_MODELS", "twitter-roberta")
    resp = client_with_model.post(
        "/api/compare",
        json={"text": "great", "model_ids": ["distilbert-sst2"]},
    )
    assert resp.status_code == 403
    assert "disabled" in resp.json()["detail"].lower()


def test_compare_allows_models_on_the_allowlist(monkeypatch, client_with_compare_models):
    monkeypatch.setenv("ENABLED_MODELS", "twitter-roberta,distilbert-sst2")
    resp = client_with_compare_models.post("/api/compare", json={"text": "great"})
    assert resp.status_code == 200
    assert len(resp.json()["results"]) == 2


def test_compare_dedupes_duplicate_model_ids(client_with_compare_models):
    # A repeated id must collapse to ONE inference/row — one anonymous request
    # can't queue the same model dozens of times by padding the list.
    resp = client_with_compare_models.post(
        "/api/compare",
        json={"text": "good", "model_ids": ["twitter-roberta", "twitter-roberta"]},
    )
    assert resp.status_code == 200
    assert [r["model_id"] for r in resp.json()["results"]] == ["twitter-roberta"]


def test_compare_rejects_over_cap_model_ids(client_with_compare_models):
    # More ids than the registry holds is rejected at the boundary (422) before
    # any weights load — a hard cap on how many inferences one request can queue.
    from app.schemas import MAX_MODEL_IDS

    too_many = [f"model-{i}" for i in range(MAX_MODEL_IDS + 1)]
    resp = client_with_compare_models.post(
        "/api/compare", json={"text": "good", "model_ids": too_many}
    )
    assert resp.status_code == 422


def test_compare_unknown_id_is_400_even_under_allowlist(monkeypatch, client_with_compare_models):
    # Guard order: an unknown id returns a truthful 400, NOT a 403 masquerading
    # as "disabled", even when the ENABLED_MODELS allowlist is active.
    monkeypatch.setenv("ENABLED_MODELS", "twitter-roberta")
    resp = client_with_compare_models.post(
        "/api/compare", json={"text": "good", "model_ids": ["not-a-real-model"]}
    )
    assert resp.status_code == 400
    assert "not-a-real-model" in resp.json()["detail"]


def test_lifespan_seeds_default_model_into_cache(monkeypatch):
    """The startup-loaded default must be seeded into model_cache so /api/compare
    reuses that one copy instead of loading a second ~500MB set of weights."""
    from fastapi.testclient import TestClient

    from app.main import app
    from app.model import SentimentModel

    # Fake a successful load without torch: is_loaded checks self._model.
    monkeypatch.setenv("SKIP_MODEL_LOAD", "0")
    monkeypatch.setattr(SentimentModel, "load", lambda self: setattr(self, "_model", object()))

    with TestClient(app) as c:
        cache = c.app.state.model_cache
        # Same object, not a second copy of the weights.
        assert cache["twitter-roberta"] is c.app.state.model