scripts/blue_team_submit.py

from __future__ import annotations

import argparse
import json
import os
from pathlib import Path

import modal

from dotenv import load_dotenv
load_dotenv()

def _require_env(name: str) -> str:
    value = os.environ.get(name, "").strip()
    if not value:
        raise ValueError(f"Missing required env var: {name}")
    return value


def _seed_dummy_dataset(app_name: str) -> None:
    seed_fn = modal.Function.from_name(app_name, "seed_dummy_dataset")
    seed = seed_fn.remote(num_images=6, image_size=224, dataset_name="dummy")
    print(f"Seeded dataset at {seed['dataset_root']}")


def main(
    models: list[str], 
    submitter: str,
    stimuli: str | None, 
    use_s3: bool = True, 
    skip_seed: bool = False
) -> None:

    _require_env("HACKATHON_MODAL_ENABLE")
    _require_env("HACKATHON_MODEL_REGISTRY")

    # Use custom stimuli if provided, otherwise use env var
    if stimuli:
        stimuli_path = Path(stimuli).resolve()
        if not stimuli_path.exists():
            raise ValueError(f"Stimuli file not found: {stimuli_path}")
        os.environ["HACKATHON_STIMULI_CATALOG"] = str(stimuli_path)
        print(f"Using custom stimuli: {stimuli_path}")
    else:
        _require_env("HACKATHON_STIMULI_CATALOG")

    # Set S3 mode if requested
    if use_s3:
        os.environ["HACKATHON_USE_S3"] = "true"
        print("S3 mode enabled - will use extract_embeddings_s3 function")

    app_name = os.environ.get("HACKATHON_MODAL_APP", "iclr2026-eval")

    # Only seed dummy dataset if not using real images
    if not skip_seed and not use_s3:
        _seed_dummy_dataset(app_name)
    elif skip_seed:
        print("Skipping dummy dataset seeding")
    elif use_s3:
        print("S3 mode: skipping dummy dataset seeding (using real images)")

    from app import submit_blue

    payload = json.dumps({"models": models})
    msg, leaderboard, pairwise = submit_blue(submitter, payload)
    print(f"Submission message: {msg}")
    print(f"Submission leaderboard: {leaderboard.tail(1).to_dict(orient='records')}")
    print(f"Submission pairwise: {pairwise.to_dict(orient='records')}")
    assert not pairwise.empty, "Pairwise table should not be empty."

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="Blue family smoke test")
    parser.add_argument(
        "--models",
        type=str,
        nargs="+",
        default=None,
        help="List of models to submit. If not provided, uses predefined model sets.",
    )
    parser.add_argument(
        "--submitter",
        type=str,
        default="test",
        help="Submitter name to use for the submission.",
    )
    parser.add_argument(
        "--stimuli",
        type=str,
        default=None,
        help="Path to custom stimuli JSONL file. If not provided, uses HACKATHON_STIMULI_CATALOG env var.",
    )
    parser.add_argument(
        "--s3",
        action="store_true",
        help="Use S3-backed datasets (requires aws-s3-credentials Modal secret).",
    )
    parser.add_argument(
        "--skip-seed",
        action="store_true",
        help="Skip seeding dummy dataset (use when testing with real images).",
    )
    args = parser.parse_args()

    models = args.models if args.models is not None else ["resnet18", "resnet34"]
    submitter = args.submitter

    main(models, submitter, args.stimuli, args.s3, args.skip_seed)