tests/test_eve.py

from pathlib import Path

from adam.eve import classify_eve_embeddings, save_eve_results
from adam.studio import StudioStore


def test_eve_sorts_good_bad_and_uncertain_examples() -> None:
    results = classify_eve_embeddings(
        ["good.png", "bad.png", "maybe.png"],
        [[1.0, 0.0], [0.0, 1.0], [0.7, 0.7]],
        [[1.0, 0.0], [0.95, 0.05]],
        [[0.0, 1.0]],
        keep_threshold=0.75,
        reject_threshold=0.25,
    )

    assert [result.suggestion for result in results] == [
        "keep", "reject", "unreviewed"
    ]
    assert results[0].match_score > results[2].match_score > results[1].match_score


def test_eve_requires_an_uncertain_confidence_band() -> None:
    try:
        classify_eve_embeddings(["one.png"], [[1.0]], [[1.0]], keep_threshold=0.4, reject_threshold=0.5)
    except ValueError as exc:
        assert "uncertain" in str(exc)
    else:
        raise AssertionError("Overlapping EVE thresholds should fail.")


def test_eve_results_are_persisted_and_mixed_decisions_apply_once(tmp_path: Path) -> None:
    dataset = tmp_path / "dataset"
    dataset.mkdir()
    good = dataset / "good.png"; good.write_bytes(b"good")
    bad = dataset / "bad.png"; bad.write_bytes(b"bad")
    results = classify_eve_embeddings(
        [good, bad], [[1.0, 0.0], [0.0, 1.0]], [[1.0, 0.0]], [[0.0, 1.0]]
    )

    record = save_eve_results(tmp_path, str(dataset), results)
    store = StudioStore(tmp_path)
    changed = store.apply_decisions(
        str(dataset), {result.path: result.suggestion for result in results}
    )

    assert record.is_file()
    assert changed == 2
    assert store.review(str(dataset)).decisions[str(good.resolve())] == "keep"
    assert store.review(str(dataset)).decisions[str(bad.resolve())] == "reject"