scripts/pipeline_smoke_test.py

from __future__ import annotations

import json
import os
from pathlib import Path

import modal


def _require_env(name: str) -> str:
    value = os.environ.get(name, "").strip()
    if not value:
        raise ValueError(f"Missing required env var: {name}")
    return value


def main() -> None:
    _require_env("HACKATHON_MODAL_ENABLE")
    _require_env("HACKATHON_MODEL_REGISTRY")
    _require_env("HACKATHON_STIMULI_CATALOG")
    data_dir = _require_env("HACKATHON_DATA_DIR")

    app_name = os.environ.get("HACKATHON_MODAL_APP", "iclr2026-eval")
    seed_fn = modal.Function.from_name(app_name, "seed_dummy_dataset")
    seed = seed_fn.remote(num_images=6, image_size=224, dataset_name="dummy")
    print(f"Seeded dataset at {seed['dataset_root']}")

    from app import submit_blue, submit_red

    blue_payload = json.dumps({"models": ["resnet18", "resnet34"]})
    red_payload = json.dumps(
        {
            "differentiating_images": [
                {"dataset_name": "dummy", "image_identifier": "images/img_0000.png"},
                {"dataset_name": "dummy", "image_identifier": "images/img_0001.png"},
            ]
        }
    )

    blue_msg, blue_leaderboard, blue_pairwise = submit_blue("pipeline-test", blue_payload)
    print("Blue message:", blue_msg)
    print("Blue leaderboard:", blue_leaderboard.tail(1).to_dict(orient="records"))
    print("Blue pairwise:", blue_pairwise.to_dict(orient="records"))

    assert not blue_leaderboard.empty, "Blue leaderboard should not be empty."
    assert not blue_pairwise.empty, "Blue pairwise table should not be empty."

    red_msg, red_leaderboard, red_pairwise = submit_red("pipeline-test", red_payload)
    print("Red message:", red_msg)
    print("Red leaderboard:", red_leaderboard.tail(1).to_dict(orient="records"))
    print("Red pairwise:", red_pairwise.to_dict(orient="records"))

    assert not red_leaderboard.empty, "Red leaderboard should not be empty."
    assert not red_pairwise.empty, "Red pairwise table should not be empty."

    for df in (blue_pairwise, red_pairwise):
        cka_vals = df["CKA"].astype(float)
        assert (cka_vals >= -1e-3).all(), "CKA should be >= 0."
        assert (cka_vals <= 1.0001).all(), "CKA should be <= 1."

    blue_path = Path(data_dir) / "blue_submissions.json"
    red_path = Path(data_dir) / "red_submissions.json"
    assert blue_path.exists(), f"Missing submission file: {blue_path}"
    assert red_path.exists(), f"Missing submission file: {red_path}"

    print("Submission pipeline smoke test complete.")


if __name__ == "__main__":
    main()