Initial commit: ICLR 2026 Representational Alignment Challenge

Public leaderboard for the Re-Align hackathon. Evaluation backend
lives in a separate private repository.

Assisted by Claude

.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+scale-hf-logo.png filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,17 @@
+auto_evals/
+venv/
+__pycache__/
+.env
+.ipynb_checkpoints
+*ipynb
+.vscode/
+.cursor/
+
+eval-queue/
+eval-results/
+eval-queue-bk/
+eval-results-bk/
+logs/
+activations/
+
+.env

.pre-commit-config.yaml

@@ -0,0 +1,53 @@
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+default_language_version:
+  python: python3
+
+ci:
+  autofix_prs: true
+  autoupdate_commit_msg: '[pre-commit.ci] pre-commit suggestions'
+  autoupdate_schedule: quarterly
+
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.3.0
+    hooks:
+      - id: check-yaml
+      - id: check-case-conflict
+      - id: detect-private-key
+      - id: check-added-large-files
+        args: ['--maxkb=1000']
+      - id: requirements-txt-fixer
+      - id: end-of-file-fixer
+      - id: trailing-whitespace
+
+  - repo: https://github.com/PyCQA/isort
+    rev: 5.12.0
+    hooks:
+      - id: isort
+        name: Format imports
+
+  - repo: https://github.com/psf/black
+    rev: 22.12.0
+    hooks:
+      - id: black
+        name: Format code
+        additional_dependencies: ['click==8.0.2']
+
+  - repo: https://github.com/charliermarsh/ruff-pre-commit
+    # Ruff version.
+    rev: 'v0.0.267'
+    hooks:
+      - id: ruff

AGENTS.md

@@ -0,0 +1,92 @@
+# AGENTS.md - Engaging (OpenMind/BCS) Cluster Guide (Project-Agnostic)
+
+This repo runs on MIT Engaging (EO) BCS resources. Use this file as the source of truth for job submission, storage, and path hygiene.
+
+## Login nodes and OS
+- Rocky 8 login nodes for BCS and MIT Rocky 8 partitions: `orcd-login001`..`orcd-login004` (or OOD Engaging Shell Access).
+- CentOS 7 login nodes only for `sched_mit_hill`: `orcd-vlogin001`..`orcd-vlogin004` (OOD Engaging Legacy Shell).
+- Do not run training or large jobs on login nodes; submit everything to Slurm.
+- Do not use interactive nodes; use `sbatch` for all compute, installs, and env verification.
+
+## BCS GPU partitions (Rocky 8)
+- `ou_bcs_high`: 4h walltime, up to 1 GPU, 32 CPUs, 1 node; reserved for interactive use only.
+- `ou_bcs_normal`: 12h walltime, up to 8 GPUs, 256 CPUs, 2 nodes; use for all batch jobs.
+- `ou_bcs_low`: 12h walltime, up to 16 GPUs, 512 CPUs, 4 nodes; preemptible, require checkpointing.
+- No Slurm account string is required for `ou_bcs_*` partitions; omit `#SBATCH --account`.
+- Always use `ou_bcs_normal` for installs, experiments, and verification jobs.
+
+## MIT partitions (Rocky 8, optional overflow)
+- `mit_normal`, `mit_normal_gpu`, `mit_preemptible` (Rocky 8 only).
+- Use `sched_mit_hill` only from CentOS 7 login nodes.
+
+## Storage layout (summary)
+- `/home/<user>`: small files, source code, 200 GB, snapshot backup.
+- `/home/<user>/orcd/pool`: medium I/O, 1 TB, no backup.
+- `/home/<user>/orcd/scratch`: fast scratch, 1 TB, no backup.
+- `/orcd/data/<PI>/001`: shared lab storage, medium I/O, no backup.
+- `/orcd/scratch/bcs/001` and `/orcd/scratch/bcs/002`: shared fast scratch, no backup.
+- `/orcd/compute/bcs/001` and `/orcd/datasets/001`: public datasets (read-only).
+
+## Path hygiene (important)
+Avoid hard-coded absolute paths. Centralize via environment variables so scripts are portable across nodes.
+
+Recommended pattern:
+- `PROJECT_ROOT=/orcd/data/<PI>/001/<user>/<project>`
+- `DATA_ROOT=/orcd/data/<PI>/001/<user>/datasets`
+- `OUTPUT_ROOT=/orcd/scratch/bcs/001/<user>/<project>`
+- `HF_HOME=/orcd/scratch/bcs/001/<user>/.cache/huggingface`
+- `TORCH_HOME=/orcd/scratch/bcs/001/<user>/.cache/torch`
+
+## Slurm template (BCS)
+```bash
+#!/bin/bash
+#SBATCH --job-name=experiment
+#SBATCH --partition=ou_bcs_normal
+#SBATCH --time=12:00:00
+#SBATCH --gpus=1
+#SBATCH --cpus-per-task=8
+#SBATCH --mem=32G
+#SBATCH --output=/orcd/data/<PI>/001/<user>/<project>/logs/%x_%j.out
+#SBATCH --error=/orcd/data/<PI>/001/<user>/<project>/logs/%x_%j.err
+
+set -euo pipefail
+set +u
+source "$HOME/miniconda3/etc/profile.d/conda.sh"
+set -u
+conda activate <env-name>
+
+export HF_HOME=/orcd/scratch/bcs/001/$USER/.cache/huggingface
+export TORCH_HOME=/orcd/scratch/bcs/001/$USER/.cache/torch
+export OUTPUT_ROOT=/orcd/scratch/bcs/001/$USER/<project>
+
+python -u <script>.py --out_dir "${OUTPUT_ROOT}/runs/run_${SLURM_JOB_ID}"
+```
+
+For sweeps, use arrays and include `%A_%a` in log paths:
+```bash
+#SBATCH --array=0-65%8
+```
+
+## Common Slurm commands
+- `sbatch <script.sh>`
+- `squeue -u $USER`
+- `sacct -j <job_id> --format=JobID,State,Elapsed,MaxRSS,AllocTRES`
+- `scontrol show job <job_id>`
+- `scancel <job_id>`
+
+## Checkpointing and preemption
+- Jobs on `ou_bcs_low` can be preempted. Always enable checkpoint writes and resume logic.
+- Store checkpoints and final outputs on `/orcd/data/<PI>/001` (longer term) or `/home/<user>/orcd/pool`.
+- Use `/orcd/scratch/bcs/001` or `/orcd/scratch/bcs/002` for intermediate outputs and large I/O.
+
+## Launch tips
+- Prefer `ou_bcs_normal` for installs and experiments; `ou_bcs_high` is often limited to one concurrent job.
+- Do large conda installs on compute nodes (Slurm job), not on login nodes.
+- Avoid `source ~/.bashrc` when using `set -u` (it can error on unbound vars); source `conda.sh` directly.
+- Use `/bin/mkdir -p` in batch scripts to avoid shell aliases or unexpected failures.
+- Set `CONDA_PKGS_DIRS=/orcd/scratch/bcs/001/$USER/.conda/pkgs` to speed installs and avoid home quota.
+- Chain setup → run with `sbatch --dependency=afterok:<jobid> <script.sh>` to ensure env is ready.
+
+## Local environment check
+- Login nodes vary; check with `hostname`.
+- Default base Python does not include GPU libraries; verify with `python -c "import torch"` via a short `sbatch` job after activating your env.

Makefile

@@ -0,0 +1,13 @@
+.PHONY: style format
+
+
+style:
+	python -m black --line-length 119 .
+	python -m isort .
+	ruff check --fix .
+
+
+quality:
+	python -m black --check --line-length 119 .
+	python -m isort --check-only .
+	ruff check .

README.md

@@ -0,0 +1,184 @@
+---
+title: Re-Align Hackathon Leaderboard
+emoji: 🥇
+colorFrom: green
+colorTo: indigo
+sdk: gradio
+app_file: app.py
+pinned: true
+license: apache-2.0
+short_description: Submit Blue/Red hackathon JSON and rank by alignment scores.
+sdk_version: 5.43.1
+tags:
+- leaderboard
+---
+
+# Overview
+
+This app runs the Re-Align Hackathon leaderboards. Blue Team submissions select model sets
+and are ranked by mean pairwise CKA. Red Team submissions select stimuli and are ranked by
+divergence (1 - avg CKA) computed across cached dummy embeddings.
+
+# Local run
+
+```bash
+conda env create -f environment.yml
+conda activate iclr2026-challenge
+python app.py
+```
+
+# Submission formats
+
+## Blue Team JSON
+
+```json
+{
+  "models": [
+    {
+      "model_name": "vit_base_patch16_224",
+      "source": "dummy_cache",
+      "model_parameters": null
+    },
+    {
+      "model_name": "resnet50",
+      "source": "dummy_cache",
+      "model_parameters": null
+    }
+  ]
+}
+```
+
+## Red Team JSON
+
+```json
+{
+  "differentiating_images": [
+    {
+      "dataset_name": "cifar100",
+      "image_identifier": "test/bear/image_0007.png"
+    },
+    {
+      "dataset_name": "imagenet1k",
+      "image_identifier": "val/n03445777/ILSVRC2012_val_00003572.JPEG"
+    }
+  ]
+}
+```
+
+# Hugging Face datasets (private only)
+
+You can optionally host the model/stimulus selections as a Hugging Face Dataset and paste the
+dataset link into the app. These datasets must be private (do not publish public submissions).
+
+## Blue Team dataset
+
+Expected column:
+- `model_names`: list of model name strings
+
+Example dataset row:
+```json
+{
+  "model_names": [
+    "vit_base_patch16_224", 
+    "resnet50", 
+    "convnext_base"
+  ]
+}
+```
+
+Create the dataset (example with `datasets` + `huggingface_hub`):
+```bash
+pip install datasets huggingface_hub
+```
+```py
+from datasets import Dataset
+from huggingface_hub import login
+
+login(token="hf_...")  # optional if you already ran `huggingface-cli login`
+
+rows = [
+    {"model_names": ["vit_base_patch16_224", "resnet50", "convnext_base"]},
+]
+ds = Dataset.from_list(rows)
+ds.push_to_hub("your-username/blue-team-submission", private=True)
+```
+
+Paste the dataset link into the app:
+```
+https://huggingface.co/datasets/your-username/blue-team-submission
+```
+
+## Red Team dataset
+
+Expected column:
+- `stimulus_id`: list of stimulus key strings (the keys shown in the app dropdown)
+
+Example dataset row:
+```json
+{
+  "stimulus_id": [
+    "cifar100:test/bear/image_0007.png", 
+    "imagenet1k:val/n03445777/ILSVRC2012_val_00003572.JPEG"
+  ]
+}
+```
+
+Create the dataset:
+```bash
+pip install datasets huggingface_hub
+```
+```py
+from datasets import Dataset
+from huggingface_hub import login
+
+login(token="hf_...")  # optional if you already ran `huggingface-cli login`
+
+rows = [
+    {
+        "stimulus_id": [
+            "cifar100:test/bear/image_0007.png",
+            "imagenet1k:val/n03445777/ILSVRC2012_val_00003572.JPEG",
+        ]
+    },
+]
+ds = Dataset.from_list(rows)
+ds.push_to_hub("your-username/red-team-submission", private=True)
+```
+
+Paste the dataset link into the app:
+```
+https://huggingface.co/datasets/your-username/red-team-submission
+```
+
+## Access token (for private datasets)
+
+1. Go to your Hugging Face account settings: `https://huggingface.co/settings/tokens`
+2. Create a new token with the `Read` scope.
+3. Copy the token (it starts with `hf_...`).
+4. Paste it into the "HuggingFace access token (optional)" textbox in the app.
+
+# Where submissions are stored
+
+Submissions are stored locally in:
+
+- `hackathon-data/blue_submissions.json`
+- `hackathon-data/red_submissions.json`
+
+You can override storage paths with:
+
+- `HACKATHON_DATA_DIR=/your/path`
+- `HACKATHON_BLUE_DATA_PATH=/your/path/blue.json`
+- `HACKATHON_RED_DATA_PATH=/your/path/red.json`
+
+# Cached embeddings
+
+Dummy cached embeddings live in `src/hackathon/data.py`. Replace `get_dummy_model_embeddings()` and
+`list_dummy_stimuli()` with real data when it is ready.
+
+See `docs/evaluation_contract.md` for the draft contract and model registry spec for real
+forward-pass evaluation.
+
+For storage layout and path hygiene conventions, see `docs/storage_layout.md`.
+
+To enable Modal-backed scoring, set `HACKATHON_MODAL_ENABLE=true` and provide
+`HACKATHON_MODEL_REGISTRY` plus `HACKATHON_STIMULI_CATALOG`.

app.py

@@ -0,0 +1,573 @@
+from dotenv import load_dotenv
+load_dotenv()
+
+import json
+import os
+
+from datetime import datetime, timezone
+from uuid import uuid4
+
+import gradio as gr
+import pandas as pd
+from datasets import load_dataset
+
+from src.about import BLUE_SUBMISSION_TEXT, GETTING_STARTED_TEXT, INTRODUCTION_TEXT, RED_SUBMISSION_TEXT, TITLE
+from src.display.css_html_js import custom_css
+from src.display.formatting import styled_error, styled_message
+from src.hackathon import (
+    append_submission,
+    list_dummy_models,
+    list_dummy_stimuli,
+    load_submissions,
+    score_blue_with_pairwise,
+    score_red_with_pairwise,
+    stimulus_key,
+)
+from src.hackathon.modal_client import fetch_volume_submissions, is_modal_enabled
+from src.hackathon.validation import (
+    MODEL_REGISTRY_ENV,
+    BLUE_MODEL_REGISTRY_ENV,
+    BLUE_TEAM_REQUIRED_MODELS,
+    STIMULI_CATALOG_ENV,
+    load_model_registry,
+    load_model_registry_specs,
+    load_stimuli_catalog,
+)
+
+
+BLUE_LEADERBOARD_COLUMNS = ["Rank", "Submitter", "Score", "Submitted"]
+BLUE_LEADERBOARD_TYPES = ["number", "str", "number", "str"]
+
+RED_LEADERBOARD_COLUMNS = ["Rank", "Submitter", "Score", "Submitted"]
+RED_LEADERBOARD_TYPES = ["number", "str", "number", "str"]
+
+PAIRWISE_COLUMNS = ["Model A", "Model B", "CKA"]
+PAIRWISE_TYPES = ["str", "str", "number"]
+
+def _load_catalogs() -> tuple[list[str], list[dict[str, str]], dict[str, dict]]:
+    if is_modal_enabled():
+        # Prefer HACKATHON_BLUE_MODEL_REGISTRY over HACKATHON_MODEL_REGISTRY
+        registry_path = os.environ.get(BLUE_MODEL_REGISTRY_ENV, "").strip()
+        if not registry_path:
+            registry_path = os.environ.get(MODEL_REGISTRY_ENV, "").strip()
+        stimuli_path = os.environ.get(STIMULI_CATALOG_ENV, "").strip()
+        if not registry_path or not stimuli_path:
+            raise ValueError(
+                "HACKATHON_MODAL_ENABLE is set but model registry or stimuli catalog is missing. "
+                "Set HACKATHON_BLUE_MODEL_REGISTRY (or HACKATHON_MODEL_REGISTRY) and HACKATHON_STIMULI_CATALOG."
+            )
+        model_names = sorted(load_model_registry(registry_path))
+        stimuli_catalog = load_stimuli_catalog(stimuli_path)
+        registry_specs = load_model_registry_specs(registry_path)
+        return model_names, stimuli_catalog, registry_specs
+
+    return list_dummy_models(), list_dummy_stimuli(), {}
+
+
+MODEL_CHOICES, STIMULI_CATALOG, BLUE_REGISTRY_SPECS = _load_catalogs()
+STIMULUS_LOOKUP = {stimulus_key(stimulus): stimulus for stimulus in STIMULI_CATALOG}
+STIMULUS_CHOICES = list(STIMULUS_LOOKUP.keys())
+
+DEFAULT_BLUE_MODELS = MODEL_CHOICES[:3]
+DEFAULT_RED_STIMULI = STIMULUS_CHOICES[:3]
+DEFAULT_SOURCE = "dummy_cache"
+
+
+def _empty_leaderboard_df(columns: list[str]) -> pd.DataFrame:
+    return pd.DataFrame(columns=columns)
+
+
+def _empty_pairwise_df() -> pd.DataFrame:
+    return pd.DataFrame(columns=PAIRWISE_COLUMNS)
+
+
+def _format_score(score: float) -> float:
+    return round(float(score), 4)
+
+
+def _build_leaderboard_df(
+    submissions: list[dict],
+    *,
+    columns: list[str],
+    count_key: str,
+    selections_key: str,
+    ascending: bool = False,
+) -> pd.DataFrame:
+    rows = []
+    for entry in submissions:
+        rows.append(
+            {
+                "Submitter": entry.get("submitter", "anonymous"),
+                "Score": _format_score(entry.get("score", 0.0)),
+                "Submitted": entry.get("submitted_time", ""),
+            }
+        )
+
+    if not rows:
+        return _empty_leaderboard_df(columns)
+
+    df = pd.DataFrame.from_records(rows, columns=columns[1:])
+    df = df.sort_values(by="Score", ascending=ascending).reset_index(drop=True)
+    df.insert(0, "Rank", range(1, len(df) + 1))
+    return df
+
+
+def _load_hf_dataset(hf_link: str, hf_token: str) -> dict | None:
+    cleaned_link = hf_link.strip()
+    if not cleaned_link:
+        return None
+    token = hf_token.strip() or None
+    return load_dataset(cleaned_link, token=token)
+
+
+def _select_dataset_split(dataset: dict) -> dict:
+    if hasattr(dataset, "column_names") and not isinstance(dataset.column_names, dict):
+        return dataset
+    return dataset.get("train") or next(iter(dataset.values()))
+
+
+def _build_blue_payload(model_names: list[str], hf_link: str, hf_token: str) -> dict:
+    if hf_link.strip() == "":
+        raise ValueError("HF dataset link is required for Blue Team submissions.")
+
+    dataset = _load_hf_dataset(hf_link, hf_token)
+    dataset_split = _select_dataset_split(dataset)
+    model_layer_list = []
+    if "model_name" in dataset_split.column_names and "layer_name" in dataset_split.column_names:
+        for model_name, layer_name in zip(dataset_split["model_name"], dataset_split["layer_name"]):
+            model_name_str = str(model_name).strip()
+            layer_name_str = str(layer_name).strip()
+            if model_name_str and layer_name_str:
+                model_layer_list.append({"model_name": model_name_str, "layer_name": layer_name_str})
+
+    return {"models": model_layer_list, "hf_dataset": hf_link.strip()}
+
+
+def _build_red_payload(stimulus_keys: list[str], hf_link: str, hf_token: str) -> dict:
+    if hf_link.strip() == "":
+        raise ValueError("HF dataset link is required for Red Team submissions.")
+
+    dataset = _load_hf_dataset(hf_link, hf_token)
+
+    dataset_split = _select_dataset_split(dataset)
+    dataset_list = []
+    if "dataset_name" in dataset_split.column_names and "image_identifier" in dataset_split.column_names:
+        for dataset_name, image_identifier in zip(dataset_split["dataset_name"], dataset_split["image_identifier"]):
+            dataset_name_str = str(dataset_name).strip()
+            image_identifier_str = str(image_identifier).strip()
+            if dataset_name_str and image_identifier_str:
+                dataset_list.append({"dataset_name": dataset_name_str, "image_identifier": image_identifier_str})
+
+    return {"differentiating_images": dataset_list, "hf_dataset": hf_link.strip()}
+
+
+def _json_dumps(payload: dict) -> str:
+    return json.dumps(payload, indent=2)
+
+
+def _parse_json_payload(payload: str) -> dict:
+    if not payload or not payload.strip():
+        raise ValueError("Paste a JSON submission payload.")
+
+    try:
+        data = json.loads(payload)
+    except json.JSONDecodeError as exc:
+        raise ValueError(f"Invalid JSON: {exc}") from exc
+
+    if not isinstance(data, dict):
+        raise ValueError("Submission JSON must be an object.")
+
+    return data
+
+
+def _extract_blue_models(payload: dict) -> tuple[list[dict], list[str]]:
+    models = payload.get("models")
+    if not isinstance(models, list):
+        raise ValueError("Blue submission must include a list of models.")
+
+    model_names = []
+    missing_layers: list[str] = []
+    layer_mismatches: list[str] = []
+    for idx, model in enumerate(models, start=1):
+        if isinstance(model, dict) and "model_name" in model:
+            name = str(model["model_name"]).strip()
+            layer_name = str(model.get("layer_name", "")).strip() or None
+        elif isinstance(model, str):
+            name = model.strip()
+            layer_name = None
+        else:
+            raise ValueError(f"Model entry {idx} must be a string or object with model_name.")
+        if name:
+            model_names.append(name)
+
+        # Validate layer_name when registry specs are available
+        if BLUE_REGISTRY_SPECS:
+            if not layer_name:
+                missing_layers.append(f"Model entry {idx} ({name}) is missing layer_name.")
+            elif name in BLUE_REGISTRY_SPECS:
+                expected_layer = BLUE_REGISTRY_SPECS[name].get("layer", "")
+                if layer_name != expected_layer:
+                    layer_mismatches.append(
+                        f"{name}: submitted layer_name '{layer_name}' "
+                        f"does not match registry layer '{expected_layer}'"
+                    )
+
+    if len(model_names) != len(set(model_names)):
+        raise ValueError("Model selections must be unique.")
+
+    if len(model_names) != BLUE_TEAM_REQUIRED_MODELS:
+        raise ValueError(
+            f"Blue team submission must contain exactly {BLUE_TEAM_REQUIRED_MODELS} "
+            f"unique models, but got {len(model_names)}."
+        )
+
+    # Validate all models are in the registry
+    registry_set = set(MODEL_CHOICES)
+    missing = [name for name in model_names if name not in registry_set]
+    if missing:
+        raise ValueError(f"Unknown models not in registry: {', '.join(missing)}")
+
+    if missing_layers:
+        raise ValueError(
+            f"Missing layer_name for {len(missing_layers)} model(s):\n"
+            + "\n".join(f"  - {m}" for m in missing_layers)
+        )
+
+    if layer_mismatches:
+        raise ValueError(
+            f"Layer name mismatch for {len(layer_mismatches)} model(s):\n"
+            + "\n".join(f"  - {m}" for m in layer_mismatches)
+        )
+
+    return models, model_names
+
+
+def _extract_red_stimuli(payload: dict) -> tuple[list[dict], list[str]]:
+    images = payload.get("differentiating_images")
+    if not isinstance(images, list):
+        raise ValueError("Red submission must include differentiating_images.")
+    if len(images) < 2:
+        raise ValueError("Select at least two stimuli.")
+
+    stimuli = []
+    stimulus_keys = []
+    for idx, item in enumerate(images, start=1):
+        if not isinstance(item, dict):
+            raise ValueError(f"Stimulus entry {idx} must be an object.")
+        dataset_name = str(item.get("dataset_name", "")).strip()
+        image_identifier = str(item.get("image_identifier", "")).strip()
+        if not dataset_name or not image_identifier:
+            raise ValueError(f"Stimulus entry {idx} must include dataset_name and image_identifier.")
+        stimulus = {"dataset_name": dataset_name, "image_identifier": image_identifier}
+        stimuli.append(stimulus)
+        stimulus_keys.append(stimulus_key(stimulus))
+
+    if len(stimulus_keys) != len(set(stimulus_keys)):
+        raise ValueError("Stimulus selections must be unique.")
+
+    missing = [key for key in stimulus_keys if key not in STIMULUS_LOOKUP]
+    if missing:
+        missing_str = ", ".join(missing)
+        raise ValueError(f"Unknown stimuli requested: {missing_str}")
+
+    return stimuli, stimulus_keys
+
+
+def _build_pairwise_df(results: list[dict]) -> pd.DataFrame:
+    if not results:
+        return _empty_pairwise_df()
+
+    df = pd.DataFrame.from_records(results, columns=PAIRWISE_COLUMNS)
+    df = df.sort_values(by="CKA", ascending=False).reset_index(drop=True)
+    return df
+
+
+def refresh_blue_leaderboard():
+    try:
+        submissions = load_submissions("blue")
+    except Exception as exc:
+        return styled_error(f"Failed to load blue submissions: {exc}"), _empty_leaderboard_df(
+            BLUE_LEADERBOARD_COLUMNS
+        )
+
+    df = _build_leaderboard_df(
+        submissions,
+        columns=BLUE_LEADERBOARD_COLUMNS,
+        count_key="num_models",
+        selections_key="model_names",
+    )
+    return styled_message(f"Loaded {len(submissions)} blue submissions."), df
+
+
+def refresh_red_leaderboard():
+    try:
+        submissions = load_submissions("red")
+    except Exception as exc:
+        return styled_error(f"Failed to load red submissions: {exc}"), _empty_leaderboard_df(RED_LEADERBOARD_COLUMNS)
+
+    df = _build_leaderboard_df(
+        submissions,
+        columns=RED_LEADERBOARD_COLUMNS,
+        count_key="num_stimuli",
+        selections_key="stimuli_keys",
+        ascending=True,  # Lower CKA = better for red team
+    )
+    return styled_message(f"Loaded {len(submissions)} red submissions."), df
+
+
+def generate_blue_json(hf_link: str, hf_token: str):
+    try:
+        payload = _build_blue_payload([], hf_link, hf_token)
+    except Exception as exc:
+        return "", styled_error(str(exc))
+
+    return _json_dumps(payload), styled_message("Generated JSON from the Blue Team form.")
+
+
+def generate_red_json(hf_link: str, hf_token: str):
+    try:
+        payload = _build_red_payload([], hf_link, hf_token)
+    except Exception as exc:
+        return "", styled_error(str(exc))
+
+    return _json_dumps(payload), styled_message("Generated JSON from the Red Team form.")
+
+
+def submit_blue(submitter: str, payload: str):
+    try:
+        data = _parse_json_payload(payload)
+        models, model_names = _extract_blue_models(data)
+        submission_id = uuid4().hex
+        submitter_name = submitter.strip() or "anonymous"
+        score, pairwise_results = score_blue_with_pairwise(
+            model_names, submission_id=submission_id, submitter=submitter_name,
+        )
+
+        submission = {
+            "submission_id": submission_id,
+            "team": "blue",
+            "submitter": submitter_name,
+            "model_names": model_names,
+            "num_models": len(model_names),
+            "score": score,
+            "submitted_time": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
+        }
+
+        submissions = append_submission("blue", submission)
+        leaderboard_df = _build_leaderboard_df(
+            submissions,
+            columns=BLUE_LEADERBOARD_COLUMNS,
+            count_key="num_models",
+            selections_key="model_names",
+        )
+        pairwise_df = _build_pairwise_df(pairwise_results)
+    except Exception as exc:
+        return (
+            styled_error(f"Blue submission failed: {exc}"),
+            _empty_leaderboard_df(BLUE_LEADERBOARD_COLUMNS),
+            _empty_pairwise_df(),
+        )
+
+    return (
+        styled_message(f"Blue submission added. Score: {_format_score(score):.4f}."),
+        leaderboard_df,
+        pairwise_df,
+    )
+
+
+def submit_red(submitter: str, payload: str):
+    try:
+        data = _parse_json_payload(payload)
+        stimuli, stimulus_keys = _extract_red_stimuli(data)
+        submission_id = uuid4().hex
+        submitter_name = submitter.strip() or "anonymous"
+        score, pairwise_results = score_red_with_pairwise(
+            stimuli, stimuli_catalog=STIMULI_CATALOG,
+            submission_id=submission_id, submitter=submitter_name,
+        )
+
+        submission = {
+            "submission_id": submission_id,
+            "team": "red",
+            "submitter": submitter_name,
+            "differentiating_images": stimuli,
+            "stimuli_keys": stimulus_keys,
+            "num_stimuli": len(stimulus_keys),
+            "score": score,
+            "submitted_time": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
+        }
+
+        submissions = append_submission("red", submission)
+        leaderboard_df = _build_leaderboard_df(
+            submissions,
+            columns=RED_LEADERBOARD_COLUMNS,
+            count_key="num_stimuli",
+            selections_key="stimuli_keys",
+            ascending=True,  # Lower CKA = better for red team
+        )
+        pairwise_df = _build_pairwise_df(pairwise_results)
+    except Exception as exc:
+        return (
+            styled_error(f"Red submission failed: {exc}"),
+            _empty_leaderboard_df(RED_LEADERBOARD_COLUMNS),
+            _empty_pairwise_df(),
+        )
+
+    return (
+        styled_message(f"Red submission added. Score: {_format_score(score):.4f}."),
+        leaderboard_df,
+        pairwise_df,
+    )
+
+
+def _sync_submissions_from_volume():
+    """Backfill any submissions saved on the Modal volume but missing from the HF dataset."""
+    if not is_modal_enabled():
+        return
+    for team in ("blue", "red"):
+        try:
+            volume_subs = fetch_volume_submissions(team=team)
+            if not volume_subs:
+                continue
+            existing = load_submissions(team)
+            existing_ids = {s.get("submission_id") for s in existing}
+            missing = [s for s in volume_subs if s.get("submission_id") not in existing_ids]
+            if missing:
+                for sub in missing:
+                    append_submission(team, sub)
+                print(f"Synced {len(missing)} missing {team} submission(s) from Modal volume.")
+        except Exception as exc:
+            print(f"Warning: failed to sync {team} submissions from Modal volume: {exc}")
+
+
+_sync_submissions_from_volume()
+
+initial_blue_status, initial_blue_leaderboard = refresh_blue_leaderboard()
+initial_red_status, initial_red_leaderboard = refresh_red_leaderboard()
+
+
+demo = gr.Blocks(css=custom_css, theme=gr.themes.Base(primary_hue="blue", neutral_hue="slate").set(body_background_fill="#0d1117", block_background_fill="#1c2128", input_background_fill="#0d1117", body_text_color="#e6edf3", block_label_text_color="#e6edf3", block_title_text_color="#e6edf3", border_color_primary="#30363d"))
+with demo:
+    gr.HTML(TITLE)
+    gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
+
+    with gr.Tabs():
+        with gr.Tab("Getting Started"):
+            gr.Markdown(GETTING_STARTED_TEXT, elem_classes="markdown-text")
+
+        with gr.Tab("Blue Team"):
+            with gr.Row(elem_classes="blue-team"):
+                with gr.Column(scale=1):
+                    gr.Markdown(BLUE_SUBMISSION_TEXT, elem_classes="markdown-text")
+                    blue_submitter_name = gr.Textbox(label="Display name", placeholder="anonymous")
+                    blue_hf_link = gr.Textbox(
+                        label="HuggingFace dataset link",
+                        placeholder="{username}/{dataset_name}",
+                    )
+                    blue_hf_token = gr.Textbox(
+                        label="HuggingFace access token",
+                        placeholder="hf_...",
+                        type="password",
+                    )
+                    blue_generate_button = gr.Button("Generate JSON", variant="secondary")
+                    blue_json = gr.Textbox(
+                        label="Blue Team JSON submission",
+                        lines=14,
+                        interactive=True,
+                    )
+                    blue_submit_button = gr.Button("Submit", variant="primary")
+                    blue_status = gr.Markdown(value=initial_blue_status)
+
+                    with gr.Accordion("Latest Blue submission pairwise CKA", open=False):
+                        blue_pairwise_table = gr.Dataframe(
+                            value=_empty_pairwise_df(),
+                            headers=PAIRWISE_COLUMNS,
+                            datatype=PAIRWISE_TYPES,
+                            row_count=8,
+                        )
+
+                with gr.Column(scale=2):
+                    gr.HTML("<h3 class='blue-team-header'>Blue Team Leaderboard</h3>")
+                    blue_leaderboard_table = gr.Dataframe(
+                        value=initial_blue_leaderboard,
+                        headers=BLUE_LEADERBOARD_COLUMNS,
+                        datatype=BLUE_LEADERBOARD_TYPES,
+                        row_count=10,
+                    )
+                    blue_refresh_button = gr.Button("Refresh Blue leaderboard", variant="secondary")
+
+        with gr.Tab("Red Team"):
+            with gr.Row(elem_classes="red-team"):
+                with gr.Column(scale=1):
+                    gr.Markdown(RED_SUBMISSION_TEXT, elem_classes="markdown-text")
+                    red_submitter_name = gr.Textbox(label="Display name", placeholder="anonymous")
+                    red_hf_link = gr.Textbox(
+                        label="HuggingFace dataset link",
+                        placeholder="{username}/{dataset_name}",
+                    )
+                    red_hf_token = gr.Textbox(
+                        label="HuggingFace access token",
+                        placeholder="hf_...",
+                        type="password",
+                    )
+                    red_generate_button = gr.Button("Generate JSON", variant="secondary")
+                    red_json = gr.Textbox(
+                        label="Red Team JSON submission",
+                        lines=14,
+                        interactive=True,
+                    )
+                    red_submit_button = gr.Button("Submit", variant="primary")
+                    red_status = gr.Markdown(value=initial_red_status)
+
+                    with gr.Accordion("Latest Red submission pairwise CKA", open=False):
+                        red_pairwise_table = gr.Dataframe(
+                            value=_empty_pairwise_df(),
+                            headers=PAIRWISE_COLUMNS,
+                            datatype=PAIRWISE_TYPES,
+                            row_count=8,
+                        )
+
+                with gr.Column(scale=2):
+                    gr.HTML("<h3 class='red-team-header'>Red Team Leaderboard</h3>")
+                    red_leaderboard_table = gr.Dataframe(
+                        value=initial_red_leaderboard,
+                        headers=RED_LEADERBOARD_COLUMNS,
+                        datatype=RED_LEADERBOARD_TYPES,
+                        row_count=10,
+                    )
+                    red_refresh_button = gr.Button("Refresh Red leaderboard", variant="secondary")
+
+    blue_generate_button.click(
+        generate_blue_json,
+        inputs=[blue_hf_link, blue_hf_token],
+        outputs=[blue_json, blue_status],
+    )
+    blue_submit_button.click(
+        submit_blue,
+        inputs=[blue_submitter_name, blue_json],
+        outputs=[blue_status, blue_leaderboard_table, blue_pairwise_table],
+    )
+    blue_refresh_button.click(
+        refresh_blue_leaderboard,
+        inputs=[],
+        outputs=[blue_status, blue_leaderboard_table],
+    )
+
+    red_generate_button.click(
+        generate_red_json,
+        inputs=[red_hf_link, red_hf_token],
+        outputs=[red_json, red_status],
+    )
+    red_submit_button.click(
+        submit_red,
+        inputs=[red_submitter_name, red_json],
+        outputs=[red_status, red_leaderboard_table, red_pairwise_table],
+    )
+    red_refresh_button.click(
+        refresh_red_leaderboard,
+        inputs=[],
+        outputs=[red_status, red_leaderboard_table],
+    )
+
+if __name__ == "__main__":
+    demo.queue(default_concurrency_limit=20).launch()

cka-data/submissions.json

@@ -0,0 +1,72 @@
+[
+  {
+    "submission_id": "651769b1660c4bf9aa1f3b6dc2b09e4e",
+    "submitter": "anonymous",
+    "embeddings": [
+      "base",
+      "base_noise_medium",
+      "base_noise_small"
+    ],
+    "num_embeddings": 3,
+    "avg_cka": 0.9819666666666667,
+    "submitted_time": "2026-01-14T18:16:10Z"
+  },
+  {
+    "submission_id": "3fe95096bd45448aa49d349b874ae35a",
+    "submitter": "anonymous",
+    "embeddings": [
+      "base",
+      "base_noise_medium",
+      "base_noise_small"
+    ],
+    "num_embeddings": 3,
+    "avg_cka": 0.9819666666666667,
+    "submitted_time": "2026-01-14T18:16:12Z"
+  },
+  {
+    "submission_id": "299d9edd3eb54fd5b44d06d080a6c32d",
+    "submitter": "anonymous",
+    "embeddings": [
+      "base",
+      "base_noise_small"
+    ],
+    "num_embeddings": 2,
+    "avg_cka": 0.9984,
+    "submitted_time": "2026-01-14T18:16:17Z"
+  },
+  {
+    "submission_id": "b5106b8d9b3b41b0950b69eb22da7c7a",
+    "submitter": "anonymous",
+    "embeddings": [
+      "base",
+      "base_noise_medium",
+      "base_noise_small",
+      "random_2"
+    ],
+    "num_embeddings": 4,
+    "avg_cka": 0.6566500000000001,
+    "submitted_time": "2026-01-14T22:07:36Z"
+  },
+  {
+    "submission_id": "ace7404ac6754b4e9ba91ce7616da300",
+    "submitter": "anonymous",
+    "embeddings": [
+      "random_2",
+      "random_3"
+    ],
+    "num_embeddings": 2,
+    "avg_cka": 0.3314,
+    "submitted_time": "2026-01-14T22:24:12Z"
+  },
+  {
+    "submission_id": "0cebb975323749d8a3c3f72fa18873e2",
+    "submitter": "anonymous",
+    "embeddings": [
+      "random_3",
+      "random_2"
+    ],
+    "num_embeddings": 2,
+    "avg_cka": 0.3314,
+    "submitted_time": "2026-01-14T22:24:34Z"
+  }
+]

condaenv.4_q_6r3k.requirements.txt

@@ -0,0 +1 @@
+-r requirements.txt

configs/blue_team_model_registry.json

@@ -0,0 +1,2963 @@
+[
+  {
+    "model_name": "aimv2_1b_patch14_224.apple_pt",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "fc_norm",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 224,
+      "crop": 224,
+      "mean": [
+        0.481,
+        0.458,
+        0.408
+      ],
+      "std": [
+        0.269,
+        0.261,
+        0.276
+      ]
+    }
+  },
+  {
+    "model_name": "bat_resnext26ts.ch_in1k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "head.global_pool",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 284,
+      "crop": 256,
+      "mean": [
+        0.485,
+        0.456,
+        0.406
+      ],
+      "std": [
+        0.229,
+        0.224,
+        0.225
+      ]
+    }
+  },
+  {
+    "model_name": "beit_base_patch16_224.in22k_ft_in22k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "fc_norm",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 249,
+      "crop": 224,
+      "mean": [
+        0.5,
+        0.5,
+        0.5
+      ],
+      "std": [
+        0.5,
+        0.5,
+        0.5
+      ]
+    }
+  },
+  {
+    "model_name": "beitv2_base_patch16_224.in1k_ft_in1k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "fc_norm",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 249,
+      "crop": 224,
+      "mean": [
+        0.485,
+        0.456,
+        0.406
+      ],
+      "std": [
+        0.229,
+        0.224,
+        0.225
+      ]
+    }
+  },
+  {
+    "model_name": "botnet26t_256.c1_in1k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "head.global_pool",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 269,
+      "crop": 256,
+      "mean": [
+        0.485,
+        0.456,
+        0.406
+      ],
+      "std": [
+        0.229,
+        0.224,
+        0.225
+      ]
+    }
+  },
+  {
+    "model_name": "caformer_b36.sail_in1k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "head.global_pool",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 224,
+      "crop": 224,
+      "mean": [
+        0.485,
+        0.456,
+        0.406
+      ],
+      "std": [
+        0.229,
+        0.224,
+        0.225
+      ]
+    }
+  },
+  {
+    "model_name": "cait_m36_384.fb_dist_in1k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "norm",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 384,
+      "crop": 384,
+      "mean": [
+        0.485,
+        0.456,
+        0.406
+      ],
+      "std": [
+        0.229,
+        0.224,
+        0.225
+      ]
+    }
+  },
+  {
+    "model_name": "coat_lite_medium.in1k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "norm4",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 249,
+      "crop": 224,
+      "mean": [
+        0.485,
+        0.456,
+        0.406
+      ],
+      "std": [
+        0.229,
+        0.224,
+        0.225
+      ]
+    }
+  },
+  {
+    "model_name": "coatnet_0_rw_224.sw_in1k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "head.global_pool",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 236,
+      "crop": 224,
+      "mean": [
+        0.5,
+        0.5,
+        0.5
+      ],
+      "std": [
+        0.5,
+        0.5,
+        0.5
+      ]
+    }
+  },
+  {
+    "model_name": "coatnext_nano_rw_224.sw_in1k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "head.global_pool",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 249,
+      "crop": 224,
+      "mean": [
+        0.5,
+        0.5,
+        0.5
+      ],
+      "std": [
+        0.5,
+        0.5,
+        0.5
+      ]
+    }
+  },
+  {
+    "model_name": "convformer_b36.sail_in1k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "head.global_pool",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 224,
+      "crop": 224,
+      "mean": [
+        0.485,
+        0.456,
+        0.406
+      ],
+      "std": [
+        0.229,
+        0.224,
+        0.225
+      ]
+    }
+  },
+  {
+    "model_name": "convit_base.fb_in1k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "norm",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 256,
+      "crop": 224,
+      "mean": [
+        0.485,
+        0.456,
+        0.406
+      ],
+      "std": [
+        0.229,
+        0.224,
+        0.225
+      ]
+    }
+  },
+  {
+    "model_name": "convmixer_1024_20_ks9_p14.in1k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "pooling",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 233,
+      "crop": 224,
+      "mean": [
+        0.485,
+        0.456,
+        0.406
+      ],
+      "std": [
+        0.229,
+        0.224,
+        0.225
+      ]
+    }
+  },
+  {
+    "model_name": "convnext_atto.d2_in1k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "head.global_pool",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 256,
+      "crop": 224,
+      "mean": [
+        0.485,
+        0.456,
+        0.406
+      ],
+      "std": [
+        0.229,
+        0.224,
+        0.225
+      ]
+    }
+  },
+  {
+    "model_name": "convnextv2_atto.fcmae",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "head.global_pool",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 256,
+      "crop": 224,
+      "mean": [
+        0.485,
+        0.456,
+        0.406
+      ],
+      "std": [
+        0.229,
+        0.224,
+        0.225
+      ]
+    }
+  },
+  {
+    "model_name": "crossvit_15_240.in1k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "norm.1",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 274,
+      "crop": 240,
+      "mean": [
+        0.485,
+        0.456,
+        0.406
+      ],
+      "std": [
+        0.229,
+        0.224,
+        0.225
+      ]
+    }
+  },
+  {
+    "model_name": "cs3darknet_focus_l.c2ns_in1k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "head.global_pool",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 289,
+      "crop": 256,
+      "mean": [
+        0.485,
+        0.456,
+        0.406
+      ],
+      "std": [
+        0.229,
+        0.224,
+        0.225
+      ]
+    }
+  },
+  {
+    "model_name": "cspdarknet53.ra_in1k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "head.global_pool",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 289,
+      "crop": 256,
+      "mean": [
+        0.485,
+        0.456,
+        0.406
+      ],
+      "std": [
+        0.229,
+        0.224,
+        0.225
+      ]
+    }
+  },
+  {
+    "model_name": "cspresnet50.ra_in1k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "head.global_pool",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 289,
+      "crop": 256,
+      "mean": [
+        0.485,
+        0.456,
+        0.406
+      ],
+      "std": [
+        0.229,
+        0.224,
+        0.225
+      ]
+    }
+  },
+  {
+    "model_name": "cspresnext50.ra_in1k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "head.global_pool",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 289,
+      "crop": 256,
+      "mean": [
+        0.485,
+        0.456,
+        0.406
+      ],
+      "std": [
+        0.229,
+        0.224,
+        0.225
+      ]
+    }
+  },
+  {
+    "model_name": "darknet53.c2ns_in1k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "head.global_pool",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 289,
+      "crop": 256,
+      "mean": [
+        0.485,
+        0.456,
+        0.406
+      ],
+      "std": [
+        0.229,
+        0.224,
+        0.225
+      ]
+    }
+  },
+  {
+    "model_name": "darknetaa53.c2ns_in1k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "head.global_pool",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 289,
+      "crop": 256,
+      "mean": [
+        0.485,
+        0.456,
+        0.406
+      ],
+      "std": [
+        0.229,
+        0.224,
+        0.225
+      ]
+    }
+  },
+  {
+    "model_name": "davit_base.msft_in1k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "head.global_pool",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 236,
+      "crop": 224,
+      "mean": [
+        0.485,
+        0.456,
+        0.406
+      ],
+      "std": [
+        0.229,
+        0.224,
+        0.225
+      ]
+    }
+  },
+  {
+    "model_name": "deit3_base_patch16_224.fb_in1k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "fc_norm",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 249,
+      "crop": 224,
+      "mean": [
+        0.485,
+        0.456,
+        0.406
+      ],
+      "std": [
+        0.229,
+        0.224,
+        0.225
+      ]
+    }
+  },
+  {
+    "model_name": "densenet121.ra_in1k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "global_pool",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 256,
+      "crop": 224,
+      "mean": [
+        0.485,
+        0.456,
+        0.406
+      ],
+      "std": [
+        0.229,
+        0.224,
+        0.225
+      ]
+    }
+  },
+  {
+    "model_name": "densenetblur121d.ra_in1k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "global_pool",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 256,
+      "crop": 224,
+      "mean": [
+        0.485,
+        0.456,
+        0.406
+      ],
+      "std": [
+        0.229,
+        0.224,
+        0.225
+      ]
+    }
+  },
+  {
+    "model_name": "dla102.in1k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "global_pool",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 256,
+      "crop": 224,
+      "mean": [
+        0.485,
+        0.456,
+        0.406
+      ],
+      "std": [
+        0.229,
+        0.224,
+        0.225
+      ]
+    }
+  },
+  {
+    "model_name": "dm_nfnet_f0.dm_in1k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "head.global_pool",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 213,
+      "crop": 192,
+      "mean": [
+        0.485,
+        0.456,
+        0.406
+      ],
+      "std": [
+        0.229,
+        0.224,
+        0.225
+      ]
+    }
+  },
+  {
+    "model_name": "dpn107.mx_in1k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "global_pool",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 256,
+      "crop": 224,
+      "mean": [
+        0.486,
+        0.459,
+        0.408
+      ],
+      "std": [
+        0.235,
+        0.235,
+        0.235
+      ]
+    }
+  },
+  {
+    "model_name": "eca_botnext26ts_256.c1_in1k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "head.global_pool",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 269,
+      "crop": 256,
+      "mean": [
+        0.485,
+        0.456,
+        0.406
+      ],
+      "std": [
+        0.229,
+        0.224,
+        0.225
+      ]
+    }
+  },
+  {
+    "model_name": "ecaresnet101d.miil_in1k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "global_pool",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 256,
+      "crop": 224,
+      "mean": [
+        0.485,
+        0.456,
+        0.406
+      ],
+      "std": [
+        0.229,
+        0.224,
+        0.225
+      ]
+    }
+  },
+  {
+    "model_name": "ecaresnetlight.miil_in1k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "global_pool",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 256,
+      "crop": 224,
+      "mean": [
+        0.485,
+        0.456,
+        0.406
+      ],
+      "std": [
+        0.229,
+        0.224,
+        0.225
+      ]
+    }
+  },
+  {
+    "model_name": "edgenext_base.in21k_ft_in1k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "head.global_pool",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 269,
+      "crop": 256,
+      "mean": [
+        0.485,
+        0.456,
+        0.406
+      ],
+      "std": [
+        0.229,
+        0.224,
+        0.225
+      ]
+    }
+  },
+  {
+    "model_name": "efficientformer_l1.snap_dist_in1k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "norm",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 236,
+      "crop": 224,
+      "mean": [
+        0.485,
+        0.456,
+        0.406
+      ],
+      "std": [
+        0.229,
+        0.224,
+        0.225
+      ]
+    }
+  },
+  {
+    "model_name": "efficientformerv2_l.snap_dist_in1k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "norm",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 236,
+      "crop": 224,
+      "mean": [
+        0.485,
+        0.456,
+        0.406
+      ],
+      "std": [
+        0.229,
+        0.224,
+        0.225
+      ]
+    }
+  },
+  {
+    "model_name": "efficientnet_b0.ra4_e3600_r224_in1k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "global_pool",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 249,
+      "crop": 224,
+      "mean": [
+        0.5,
+        0.5,
+        0.5
+      ],
+      "std": [
+        0.5,
+        0.5,
+        0.5
+      ]
+    }
+  },
+  {
+    "model_name": "efficientnetv2_rw_m.agc_in1k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "global_pool",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 320,
+      "crop": 320,
+      "mean": [
+        0.485,
+        0.456,
+        0.406
+      ],
+      "std": [
+        0.229,
+        0.224,
+        0.225
+      ]
+    }
+  },
+  {
+    "model_name": "efficientvit_b0.r224_in1k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "head.global_pool",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 236,
+      "crop": 224,
+      "mean": [
+        0.485,
+        0.456,
+        0.406
+      ],
+      "std": [
+        0.229,
+        0.224,
+        0.225
+      ]
+    }
+  },
+  {
+    "model_name": "ese_vovnet19b_dw.ra_in1k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "head.global_pool",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 256,
+      "crop": 224,
+      "mean": [
+        0.485,
+        0.456,
+        0.406
+      ],
+      "std": [
+        0.229,
+        0.224,
+        0.225
+      ]
+    }
+  },
+  {
+    "model_name": "eva02_base_patch14_224.mim_in22k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "fc_norm",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 249,
+      "crop": 224,
+      "mean": [
+        0.481,
+        0.458,
+        0.408
+      ],
+      "std": [
+        0.269,
+        0.261,
+        0.276
+      ]
+    }
+  },
+  {
+    "model_name": "fastvit_ma36.apple_dist_in1k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "head.global_pool",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 269,
+      "crop": 256,
+      "mean": [
+        0.485,
+        0.456,
+        0.406
+      ],
+      "std": [
+        0.229,
+        0.224,
+        0.225
+      ]
+    }
+  },
+  {
+    "model_name": "fbnetc_100.rmsp_in1k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "global_pool",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 256,
+      "crop": 224,
+      "mean": [
+        0.485,
+        0.456,
+        0.406
+      ],
+      "std": [
+        0.229,
+        0.224,
+        0.225
+      ]
+    }
+  },
+  {
+    "model_name": "fbnetv3_b.ra2_in1k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "global_pool",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 236,
+      "crop": 224,
+      "mean": [
+        0.485,
+        0.456,
+        0.406
+      ],
+      "std": [
+        0.229,
+        0.224,
+        0.225
+      ]
+    }
+  },
+  {
+    "model_name": "flexivit_base.1000ep_in21k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "fc_norm",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 253,
+      "crop": 240,
+      "mean": [
+        0.5,
+        0.5,
+        0.5
+      ],
+      "std": [
+        0.5,
+        0.5,
+        0.5
+      ]
+    }
+  },
+  {
+    "model_name": "focalnet_base_lrf.ms_in1k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "head.global_pool",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 249,
+      "crop": 224,
+      "mean": [
+        0.485,
+        0.456,
+        0.406
+      ],
+      "std": [
+        0.229,
+        0.224,
+        0.225
+      ]
+    }
+  },
+  {
+    "model_name": "gc_efficientnetv2_rw_t.agc_in1k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "global_pool",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 224,
+      "crop": 224,
+      "mean": [
+        0.485,
+        0.456,
+        0.406
+      ],
+      "std": [
+        0.229,
+        0.224,
+        0.225
+      ]
+    }
+  },
+  {
+    "model_name": "gcresnet33ts.ra2_in1k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "head.global_pool",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 284,
+      "crop": 256,
+      "mean": [
+        0.485,
+        0.456,
+        0.406
+      ],
+      "std": [
+        0.229,
+        0.224,
+        0.225
+      ]
+    }
+  },
+  {
+    "model_name": "gcresnext26ts.ch_in1k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "head.global_pool",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 284,
+      "crop": 256,
+      "mean": [
+        0.485,
+        0.456,
+        0.406
+      ],
+      "std": [
+        0.229,
+        0.224,
+        0.225
+      ]
+    }
+  },
+  {
+    "model_name": "gcvit_base.in1k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "head.global_pool",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 256,
+      "crop": 224,
+      "mean": [
+        0.485,
+        0.456,
+        0.406
+      ],
+      "std": [
+        0.229,
+        0.224,
+        0.225
+      ]
+    }
+  },
+  {
+    "model_name": "gernet_l.idstcv_in1k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "head.global_pool",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 293,
+      "crop": 256,
+      "mean": [
+        0.485,
+        0.456,
+        0.406
+      ],
+      "std": [
+        0.229,
+        0.224,
+        0.225
+      ]
+    }
+  },
+  {
+    "model_name": "ghostnet_100.in1k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "global_pool",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 256,
+      "crop": 224,
+      "mean": [
+        0.485,
+        0.456,
+        0.406
+      ],
+      "std": [
+        0.229,
+        0.224,
+        0.225
+      ]
+    }
+  },
+  {
+    "model_name": "ghostnetv2_100.in1k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "global_pool",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 256,
+      "crop": 224,
+      "mean": [
+        0.485,
+        0.456,
+        0.406
+      ],
+      "std": [
+        0.229,
+        0.224,
+        0.225
+      ]
+    }
+  },
+  {
+    "model_name": "gmixer_24_224.ra3_in1k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "norm",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 256,
+      "crop": 224,
+      "mean": [
+        0.485,
+        0.456,
+        0.406
+      ],
+      "std": [
+        0.229,
+        0.224,
+        0.225
+      ]
+    }
+  },
+  {
+    "model_name": "gmlp_s16_224.ra3_in1k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "norm",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 256,
+      "crop": 224,
+      "mean": [
+        0.5,
+        0.5,
+        0.5
+      ],
+      "std": [
+        0.5,
+        0.5,
+        0.5
+      ]
+    }
+  },
+  {
+    "model_name": "halo2botnet50ts_256.a1h_in1k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "head.global_pool",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 269,
+      "crop": 256,
+      "mean": [
+        0.485,
+        0.456,
+        0.406
+      ],
+      "std": [
+        0.229,
+        0.224,
+        0.225
+      ]
+    }
+  },
+  {
+    "model_name": "halonet26t.a1h_in1k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "head.global_pool",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 269,
+      "crop": 256,
+      "mean": [
+        0.485,
+        0.456,
+        0.406
+      ],
+      "std": [
+        0.229,
+        0.224,
+        0.225
+      ]
+    }
+  },
+  {
+    "model_name": "haloregnetz_b.ra3_in1k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "head.global_pool",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 238,
+      "crop": 224,
+      "mean": [
+        0.5,
+        0.5,
+        0.5
+      ],
+      "std": [
+        0.5,
+        0.5,
+        0.5
+      ]
+    }
+  },
+  {
+    "model_name": "hardcorenas_a.miil_green_in1k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "global_pool",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 256,
+      "crop": 224,
+      "mean": [
+        0.485,
+        0.456,
+        0.406
+      ],
+      "std": [
+        0.229,
+        0.224,
+        0.225
+      ]
+    }
+  },
+  {
+    "model_name": "hgnet_base.ssld_in1k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "head.global_pool",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 232,
+      "crop": 224,
+      "mean": [
+        0.485,
+        0.456,
+        0.406
+      ],
+      "std": [
+        0.229,
+        0.224,
+        0.225
+      ]
+    }
+  },
+  {
+    "model_name": "hgnetv2_b0.ssld_stage1_in22k_in1k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "head.global_pool",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 232,
+      "crop": 224,
+      "mean": [
+        0.485,
+        0.456,
+        0.406
+      ],
+      "std": [
+        0.229,
+        0.224,
+        0.225
+      ]
+    }
+  },
+  {
+    "model_name": "hiera_base_224.mae",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "head.norm",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 249,
+      "crop": 224,
+      "mean": [
+        0.485,
+        0.456,
+        0.406
+      ],
+      "std": [
+        0.229,
+        0.224,
+        0.225
+      ]
+    }
+  },
+  {
+    "model_name": "hrnet_w18.ms_aug_in1k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "global_pool",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 236,
+      "crop": 224,
+      "mean": [
+        0.485,
+        0.456,
+        0.406
+      ],
+      "std": [
+        0.229,
+        0.224,
+        0.225
+      ]
+    }
+  },
+  {
+    "model_name": "inception_next_atto.sail_in1k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "head.global_pool",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 256,
+      "crop": 224,
+      "mean": [
+        0.485,
+        0.456,
+        0.406
+      ],
+      "std": [
+        0.229,
+        0.224,
+        0.225
+      ]
+    }
+  },
+  {
+    "model_name": "lambda_resnet26rpt_256.c1_in1k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "head.global_pool",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 272,
+      "crop": 256,
+      "mean": [
+        0.485,
+        0.456,
+        0.406
+      ],
+      "std": [
+        0.229,
+        0.224,
+        0.225
+      ]
+    }
+  },
+  {
+    "model_name": "lamhalobotnet50ts_256.a1h_in1k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "head.global_pool",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 269,
+      "crop": 256,
+      "mean": [
+        0.485,
+        0.456,
+        0.406
+      ],
+      "std": [
+        0.229,
+        0.224,
+        0.225
+      ]
+    }
+  },
+  {
+    "model_name": "lcnet_050.ra2_in1k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "global_pool",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 256,
+      "crop": 224,
+      "mean": [
+        0.485,
+        0.456,
+        0.406
+      ],
+      "std": [
+        0.229,
+        0.224,
+        0.225
+      ]
+    }
+  },
+  {
+    "model_name": "legacy_senet154.in1k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "global_pool",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 256,
+      "crop": 224,
+      "mean": [
+        0.485,
+        0.456,
+        0.406
+      ],
+      "std": [
+        0.229,
+        0.224,
+        0.225
+      ]
+    }
+  },
+  {
+    "model_name": "levit_128.fb_dist_in1k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "head.bn",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 249,
+      "crop": 224,
+      "mean": [
+        0.485,
+        0.456,
+        0.406
+      ],
+      "std": [
+        0.229,
+        0.224,
+        0.225
+      ]
+    }
+  },
+  {
+    "model_name": "mambaout_base.in1k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "head.norm",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 224,
+      "crop": 224,
+      "mean": [
+        0.485,
+        0.456,
+        0.406
+      ],
+      "std": [
+        0.229,
+        0.224,
+        0.225
+      ]
+    }
+  },
+  {
+    "model_name": "maxvit_base_tf_224.in1k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "head.global_pool",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 236,
+      "crop": 224,
+      "mean": [
+        0.485,
+        0.456,
+        0.406
+      ],
+      "std": [
+        0.229,
+        0.224,
+        0.225
+      ]
+    }
+  },
+  {
+    "model_name": "maxxvit_rmlp_nano_rw_256.sw_in1k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "head.global_pool",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 269,
+      "crop": 256,
+      "mean": [
+        0.5,
+        0.5,
+        0.5
+      ],
+      "std": [
+        0.5,
+        0.5,
+        0.5
+      ]
+    }
+  },
+  {
+    "model_name": "maxxvitv2_nano_rw_256.sw_in1k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "head.global_pool",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 269,
+      "crop": 256,
+      "mean": [
+        0.5,
+        0.5,
+        0.5
+      ],
+      "std": [
+        0.5,
+        0.5,
+        0.5
+      ]
+    }
+  },
+  {
+    "model_name": "mixer_b16_224.goog_in21k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "norm",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 256,
+      "crop": 224,
+      "mean": [
+        0.5,
+        0.5,
+        0.5
+      ],
+      "std": [
+        0.5,
+        0.5,
+        0.5
+      ]
+    }
+  },
+  {
+    "model_name": "mixnet_l.ft_in1k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "global_pool",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 256,
+      "crop": 224,
+      "mean": [
+        0.485,
+        0.456,
+        0.406
+      ],
+      "std": [
+        0.229,
+        0.224,
+        0.225
+      ]
+    }
+  },
+  {
+    "model_name": "mnasnet_100.rmsp_in1k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "global_pool",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 256,
+      "crop": 224,
+      "mean": [
+        0.485,
+        0.456,
+        0.406
+      ],
+      "std": [
+        0.229,
+        0.224,
+        0.225
+      ]
+    }
+  },
+  {
+    "model_name": "mobilenet_edgetpu_v2_m.ra4_e3600_r224_in1k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "global_pool",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 249,
+      "crop": 224,
+      "mean": [
+        0.5,
+        0.5,
+        0.5
+      ],
+      "std": [
+        0.5,
+        0.5,
+        0.5
+      ]
+    }
+  },
+  {
+    "model_name": "mobilenetv1_100.ra4_e3600_r224_in1k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "global_pool",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 256,
+      "crop": 224,
+      "mean": [
+        0.5,
+        0.5,
+        0.5
+      ],
+      "std": [
+        0.5,
+        0.5,
+        0.5
+      ]
+    }
+  },
+  {
+    "model_name": "mobileone_s0.apple_in1k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "head.global_pool",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 256,
+      "crop": 224,
+      "mean": [
+        0.485,
+        0.456,
+        0.406
+      ],
+      "std": [
+        0.229,
+        0.224,
+        0.225
+      ]
+    }
+  },
+  {
+    "model_name": "mobilevit_s.cvnets_in1k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "head.global_pool",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 284,
+      "crop": 256,
+      "mean": [
+        0.0,
+        0.0,
+        0.0
+      ],
+      "std": [
+        1.0,
+        1.0,
+        1.0
+      ]
+    }
+  },
+  {
+    "model_name": "mobilevitv2_050.cvnets_in1k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "head.global_pool",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 288,
+      "crop": 256,
+      "mean": [
+        0.0,
+        0.0,
+        0.0
+      ],
+      "std": [
+        1.0,
+        1.0,
+        1.0
+      ]
+    }
+  },
+  {
+    "model_name": "mvitv2_base.fb_in1k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "norm",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 249,
+      "crop": 224,
+      "mean": [
+        0.485,
+        0.456,
+        0.406
+      ],
+      "std": [
+        0.229,
+        0.224,
+        0.225
+      ]
+    }
+  },
+  {
+    "model_name": "nasnetalarge.tf_in1k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "global_pool",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 363,
+      "crop": 331,
+      "mean": [
+        0.5,
+        0.5,
+        0.5
+      ],
+      "std": [
+        0.5,
+        0.5,
+        0.5
+      ]
+    }
+  },
+  {
+    "model_name": "nest_base_jx.goog_in1k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "global_pool",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 256,
+      "crop": 224,
+      "mean": [
+        0.485,
+        0.456,
+        0.406
+      ],
+      "std": [
+        0.229,
+        0.224,
+        0.225
+      ]
+    }
+  },
+  {
+    "model_name": "nextvit_base.bd_in1k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "head.global_pool",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 236,
+      "crop": 224,
+      "mean": [
+        0.485,
+        0.456,
+        0.406
+      ],
+      "std": [
+        0.229,
+        0.224,
+        0.225
+      ]
+    }
+  },
+  {
+    "model_name": "nf_regnet_b1.ra2_in1k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "head.global_pool",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 284,
+      "crop": 256,
+      "mean": [
+        0.485,
+        0.456,
+        0.406
+      ],
+      "std": [
+        0.229,
+        0.224,
+        0.225
+      ]
+    }
+  },
+  {
+    "model_name": "nfnet_l0.ra2_in1k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "head.global_pool",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 249,
+      "crop": 224,
+      "mean": [
+        0.485,
+        0.456,
+        0.406
+      ],
+      "std": [
+        0.229,
+        0.224,
+        0.225
+      ]
+    }
+  },
+  {
+    "model_name": "pit_b_224.in1k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "norm",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 249,
+      "crop": 224,
+      "mean": [
+        0.485,
+        0.456,
+        0.406
+      ],
+      "std": [
+        0.229,
+        0.224,
+        0.225
+      ]
+    }
+  },
+  {
+    "model_name": "pnasnet5large.tf_in1k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "global_pool",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 363,
+      "crop": 331,
+      "mean": [
+        0.5,
+        0.5,
+        0.5
+      ],
+      "std": [
+        0.5,
+        0.5,
+        0.5
+      ]
+    }
+  },
+  {
+    "model_name": "poolformer_m36.sail_in1k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "head.global_pool",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 236,
+      "crop": 224,
+      "mean": [
+        0.485,
+        0.456,
+        0.406
+      ],
+      "std": [
+        0.229,
+        0.224,
+        0.225
+      ]
+    }
+  },
+  {
+    "model_name": "poolformerv2_m36.sail_in1k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "head.global_pool",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 224,
+      "crop": 224,
+      "mean": [
+        0.485,
+        0.456,
+        0.406
+      ],
+      "std": [
+        0.229,
+        0.224,
+        0.225
+      ]
+    }
+  },
+  {
+    "model_name": "pvt_v2_b0.in1k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "stages.3.norm",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 249,
+      "crop": 224,
+      "mean": [
+        0.485,
+        0.456,
+        0.406
+      ],
+      "std": [
+        0.229,
+        0.224,
+        0.225
+      ]
+    }
+  },
+  {
+    "model_name": "rdnet_base.nv_in1k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "head.global_pool",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 249,
+      "crop": 224,
+      "mean": [
+        0.485,
+        0.456,
+        0.406
+      ],
+      "std": [
+        0.229,
+        0.224,
+        0.225
+      ]
+    }
+  },
+  {
+    "model_name": "regnetv_040.ra3_in1k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "head.global_pool",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 236,
+      "crop": 224,
+      "mean": [
+        0.485,
+        0.456,
+        0.406
+      ],
+      "std": [
+        0.229,
+        0.224,
+        0.225
+      ]
+    }
+  },
+  {
+    "model_name": "regnetx_002.pycls_in1k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "head.global_pool",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 256,
+      "crop": 224,
+      "mean": [
+        0.485,
+        0.456,
+        0.406
+      ],
+      "std": [
+        0.229,
+        0.224,
+        0.225
+      ]
+    }
+  },
+  {
+    "model_name": "regnety_002.pycls_in1k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "head.global_pool",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 256,
+      "crop": 224,
+      "mean": [
+        0.485,
+        0.456,
+        0.406
+      ],
+      "std": [
+        0.229,
+        0.224,
+        0.225
+      ]
+    }
+  },
+  {
+    "model_name": "regnetz_040.ra3_in1k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "head.global_pool",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 256,
+      "crop": 256,
+      "mean": [
+        0.485,
+        0.456,
+        0.406
+      ],
+      "std": [
+        0.229,
+        0.224,
+        0.225
+      ]
+    }
+  },
+  {
+    "model_name": "repghostnet_050.in1k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "global_pool",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 256,
+      "crop": 224,
+      "mean": [
+        0.485,
+        0.456,
+        0.406
+      ],
+      "std": [
+        0.229,
+        0.224,
+        0.225
+      ]
+    }
+  },
+  {
+    "model_name": "repvgg_a0.rvgg_in1k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "head.global_pool",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 256,
+      "crop": 224,
+      "mean": [
+        0.485,
+        0.456,
+        0.406
+      ],
+      "std": [
+        0.229,
+        0.224,
+        0.225
+      ]
+    }
+  },
+  {
+    "model_name": "repvit_m0_9.dist_300e_in1k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "head.head.bn",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 236,
+      "crop": 224,
+      "mean": [
+        0.485,
+        0.456,
+        0.406
+      ],
+      "std": [
+        0.229,
+        0.224,
+        0.225
+      ]
+    }
+  },
+  {
+    "model_name": "res2net101_26w_4s.in1k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "global_pool",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 256,
+      "crop": 224,
+      "mean": [
+        0.485,
+        0.456,
+        0.406
+      ],
+      "std": [
+        0.229,
+        0.224,
+        0.225
+      ]
+    }
+  },
+  {
+    "model_name": "resmlp_12_224.fb_dino",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "norm",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 256,
+      "crop": 224,
+      "mean": [
+        0.485,
+        0.456,
+        0.406
+      ],
+      "std": [
+        0.229,
+        0.224,
+        0.225
+      ]
+    }
+  },
+  {
+    "model_name": "resnest101e.in1k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "global_pool",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 293,
+      "crop": 256,
+      "mean": [
+        0.485,
+        0.456,
+        0.406
+      ],
+      "std": [
+        0.229,
+        0.224,
+        0.225
+      ]
+    }
+  },
+  {
+    "model_name": "resnet101.a1_in1k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "global_pool",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 236,
+      "crop": 224,
+      "mean": [
+        0.485,
+        0.456,
+        0.406
+      ],
+      "std": [
+        0.229,
+        0.224,
+        0.225
+      ]
+    }
+  },
+  {
+    "model_name": "resnetaa101d.sw_in12k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "global_pool",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 236,
+      "crop": 224,
+      "mean": [
+        0.485,
+        0.456,
+        0.406
+      ],
+      "std": [
+        0.229,
+        0.224,
+        0.225
+      ]
+    }
+  },
+  {
+    "model_name": "resnetblur50.bt_in1k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "global_pool",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 256,
+      "crop": 224,
+      "mean": [
+        0.485,
+        0.456,
+        0.406
+      ],
+      "std": [
+        0.229,
+        0.224,
+        0.225
+      ]
+    }
+  },
+  {
+    "model_name": "resnetrs101.tf_in1k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "global_pool",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 204,
+      "crop": 192,
+      "mean": [
+        0.485,
+        0.456,
+        0.406
+      ],
+      "std": [
+        0.229,
+        0.224,
+        0.225
+      ]
+    }
+  },
+  {
+    "model_name": "resnetv2_101.a1h_in1k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "head.global_pool",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 236,
+      "crop": 224,
+      "mean": [
+        0.5,
+        0.5,
+        0.5
+      ],
+      "std": [
+        0.5,
+        0.5,
+        0.5
+      ]
+    }
+  },
+  {
+    "model_name": "resnext101_32x16d.fb_ssl_yfcc100m_ft_in1k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "global_pool",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 256,
+      "crop": 224,
+      "mean": [
+        0.485,
+        0.456,
+        0.406
+      ],
+      "std": [
+        0.229,
+        0.224,
+        0.225
+      ]
+    }
+  },
+  {
+    "model_name": "rexnet_100.nav_in1k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "head.global_pool",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 256,
+      "crop": 224,
+      "mean": [
+        0.485,
+        0.456,
+        0.406
+      ],
+      "std": [
+        0.229,
+        0.224,
+        0.225
+      ]
+    }
+  },
+  {
+    "model_name": "rexnetr_200.sw_in12k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "head.global_pool",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 236,
+      "crop": 224,
+      "mean": [
+        0.485,
+        0.456,
+        0.406
+      ],
+      "std": [
+        0.229,
+        0.224,
+        0.225
+      ]
+    }
+  },
+  {
+    "model_name": "sam2_hiera_base_plus.fb_r896",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "head.norm",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 896,
+      "crop": 896,
+      "mean": [
+        0.485,
+        0.456,
+        0.406
+      ],
+      "std": [
+        0.229,
+        0.224,
+        0.225
+      ]
+    }
+  },
+  {
+    "model_name": "samvit_base_patch16.sa1b",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "head.global_pool",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 1024,
+      "crop": 1024,
+      "mean": [
+        0.485,
+        0.456,
+        0.406
+      ],
+      "std": [
+        0.229,
+        0.224,
+        0.225
+      ]
+    }
+  },
+  {
+    "model_name": "sebotnet33ts_256.a1h_in1k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "head.global_pool",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 272,
+      "crop": 256,
+      "mean": [
+        0.485,
+        0.456,
+        0.406
+      ],
+      "std": [
+        0.229,
+        0.224,
+        0.225
+      ]
+    }
+  },
+  {
+    "model_name": "sehalonet33ts.ra2_in1k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "head.global_pool",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 272,
+      "crop": 256,
+      "mean": [
+        0.485,
+        0.456,
+        0.406
+      ],
+      "std": [
+        0.229,
+        0.224,
+        0.225
+      ]
+    }
+  },
+  {
+    "model_name": "selecsls42b.in1k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "global_pool",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 256,
+      "crop": 224,
+      "mean": [
+        0.485,
+        0.456,
+        0.406
+      ],
+      "std": [
+        0.229,
+        0.224,
+        0.225
+      ]
+    }
+  },
+  {
+    "model_name": "semnasnet_075.rmsp_in1k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "global_pool",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 256,
+      "crop": 224,
+      "mean": [
+        0.485,
+        0.456,
+        0.406
+      ],
+      "std": [
+        0.229,
+        0.224,
+        0.225
+      ]
+    }
+  },
+  {
+    "model_name": "senet154.gluon_in1k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "global_pool",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 256,
+      "crop": 224,
+      "mean": [
+        0.485,
+        0.456,
+        0.406
+      ],
+      "std": [
+        0.229,
+        0.224,
+        0.225
+      ]
+    }
+  },
+  {
+    "model_name": "sequencer2d_l.in1k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "head.global_pool",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 256,
+      "crop": 224,
+      "mean": [
+        0.485,
+        0.456,
+        0.406
+      ],
+      "std": [
+        0.229,
+        0.224,
+        0.225
+      ]
+    }
+  },
+  {
+    "model_name": "seresnet152d.ra2_in1k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "global_pool",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 269,
+      "crop": 256,
+      "mean": [
+        0.485,
+        0.456,
+        0.406
+      ],
+      "std": [
+        0.229,
+        0.224,
+        0.225
+      ]
+    }
+  },
+  {
+    "model_name": "seresnext101_32x4d.gluon_in1k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "global_pool",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 256,
+      "crop": 224,
+      "mean": [
+        0.485,
+        0.456,
+        0.406
+      ],
+      "std": [
+        0.229,
+        0.224,
+        0.225
+      ]
+    }
+  },
+  {
+    "model_name": "seresnextaa101d_32x8d.ah_in1k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "global_pool",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 236,
+      "crop": 224,
+      "mean": [
+        0.485,
+        0.456,
+        0.406
+      ],
+      "std": [
+        0.229,
+        0.224,
+        0.225
+      ]
+    }
+  },
+  {
+    "model_name": "skresnet18.ra_in1k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "global_pool",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 256,
+      "crop": 224,
+      "mean": [
+        0.485,
+        0.456,
+        0.406
+      ],
+      "std": [
+        0.229,
+        0.224,
+        0.225
+      ]
+    }
+  },
+  {
+    "model_name": "skresnext50_32x4d.ra_in1k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "global_pool",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 256,
+      "crop": 224,
+      "mean": [
+        0.485,
+        0.456,
+        0.406
+      ],
+      "std": [
+        0.229,
+        0.224,
+        0.225
+      ]
+    }
+  },
+  {
+    "model_name": "spnasnet_100.rmsp_in1k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "global_pool",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 256,
+      "crop": 224,
+      "mean": [
+        0.485,
+        0.456,
+        0.406
+      ],
+      "std": [
+        0.229,
+        0.224,
+        0.225
+      ]
+    }
+  },
+  {
+    "model_name": "swin_base_patch4_window12_384.ms_in1k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "head.global_pool",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 384,
+      "crop": 384,
+      "mean": [
+        0.485,
+        0.456,
+        0.406
+      ],
+      "std": [
+        0.229,
+        0.224,
+        0.225
+      ]
+    }
+  },
+  {
+    "model_name": "swinv2_base_window12_192.ms_in22k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "head.global_pool",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 213,
+      "crop": 192,
+      "mean": [
+        0.485,
+        0.456,
+        0.406
+      ],
+      "std": [
+        0.229,
+        0.224,
+        0.225
+      ]
+    }
+  },
+  {
+    "model_name": "test_byobnet.r160_in1k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "head.global_pool",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 168,
+      "crop": 160,
+      "mean": [
+        0.485,
+        0.456,
+        0.406
+      ],
+      "std": [
+        0.229,
+        0.224,
+        0.225
+      ]
+    }
+  },
+  {
+    "model_name": "tf_efficientnet_b0.aa_in1k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "global_pool",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 256,
+      "crop": 224,
+      "mean": [
+        0.485,
+        0.456,
+        0.406
+      ],
+      "std": [
+        0.229,
+        0.224,
+        0.225
+      ]
+    }
+  },
+  {
+    "model_name": "tiny_vit_11m_224.dist_in22k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "head.global_pool",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 236,
+      "crop": 224,
+      "mean": [
+        0.485,
+        0.456,
+        0.406
+      ],
+      "std": [
+        0.229,
+        0.224,
+        0.225
+      ]
+    }
+  },
+  {
+    "model_name": "tinynet_a.in1k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "global_pool",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 219,
+      "crop": 192,
+      "mean": [
+        0.485,
+        0.456,
+        0.406
+      ],
+      "std": [
+        0.229,
+        0.224,
+        0.225
+      ]
+    }
+  },
+  {
+    "model_name": "tnt_s_patch16_224",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "norm",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 249,
+      "crop": 224,
+      "mean": [
+        0.5,
+        0.5,
+        0.5
+      ],
+      "std": [
+        0.5,
+        0.5,
+        0.5
+      ]
+    }
+  },
+  {
+    "model_name": "tresnet_l.miil_in1k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "head.global_pool",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 256,
+      "crop": 224,
+      "mean": [
+        0.0,
+        0.0,
+        0.0
+      ],
+      "std": [
+        1.0,
+        1.0,
+        1.0
+      ]
+    }
+  },
+  {
+    "model_name": "twins_pcpvt_base.in1k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "norm",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 249,
+      "crop": 224,
+      "mean": [
+        0.485,
+        0.456,
+        0.406
+      ],
+      "std": [
+        0.229,
+        0.224,
+        0.225
+      ]
+    }
+  },
+  {
+    "model_name": "vgg11.tv_in1k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "head.global_pool",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 256,
+      "crop": 224,
+      "mean": [
+        0.485,
+        0.456,
+        0.406
+      ],
+      "std": [
+        0.229,
+        0.224,
+        0.225
+      ]
+    }
+  },
+  {
+    "model_name": "visformer_small.in1k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "global_pool",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 249,
+      "crop": 224,
+      "mean": [
+        0.485,
+        0.456,
+        0.406
+      ],
+      "std": [
+        0.229,
+        0.224,
+        0.225
+      ]
+    }
+  },
+  {
+    "model_name": "vit_base_mci_224.apple_mclip",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "fc_norm",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 249,
+      "crop": 224,
+      "mean": [
+        0.0,
+        0.0,
+        0.0
+      ],
+      "std": [
+        1.0,
+        1.0,
+        1.0
+      ]
+    }
+  },
+  {
+    "model_name": "vitamin_base_224.datacomp1b_clip",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "fc_norm",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 249,
+      "crop": 224,
+      "mean": [
+        0.481,
+        0.458,
+        0.408
+      ],
+      "std": [
+        0.269,
+        0.261,
+        0.276
+      ]
+    }
+  },
+  {
+    "model_name": "volo_d1_224.sail_in1k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "norm",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 233,
+      "crop": 224,
+      "mean": [
+        0.485,
+        0.456,
+        0.406
+      ],
+      "std": [
+        0.229,
+        0.224,
+        0.225
+      ]
+    }
+  },
+  {
+    "model_name": "wide_resnet101_2.tv2_in1k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "global_pool",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 201,
+      "crop": 176,
+      "mean": [
+        0.485,
+        0.456,
+        0.406
+      ],
+      "std": [
+        0.229,
+        0.224,
+        0.225
+      ]
+    }
+  },
+  {
+    "model_name": "xception41.tf_in1k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "head.global_pool",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 331,
+      "crop": 299,
+      "mean": [
+        0.5,
+        0.5,
+        0.5
+      ],
+      "std": [
+        0.5,
+        0.5,
+        0.5
+      ]
+    }
+  },
+  {
+    "model_name": "xcit_large_24_p16_224.fb_dist_in1k",
+    "source": "timm",
+    "weights": "imagenet",
+    "layer": "norm",
+    "embedding": "flatten",
+    "preprocess": {
+      "resize": 224,
+      "crop": 224,
+      "mean": [
+        0.485,
+        0.456,
+        0.406
+      ],
+      "std": [
+        0.229,
+        0.224,
+        0.225
+      ]
+    }
+  }
+]

configs/dataset_roots.example.json

@@ -0,0 +1,4 @@
+{
+  "cifar100": "${DATA_ROOT}/cifar100",
+  "imagenet1k": "${DATA_ROOT}/imagenet1k"
+}

configs/one_model_per_family.json

@@ -0,0 +1,143 @@
+[
+  "aimv2_1b_patch14_224.apple_pt",
+  "bat_resnext26ts.ch_in1k",
+  "beit_base_patch16_224.in22k_ft_in22k",
+  "beitv2_base_patch16_224.in1k_ft_in1k",
+  "botnet26t_256.c1_in1k",
+  "caformer_b36.sail_in1k",
+  "cait_m36_384.fb_dist_in1k",
+  "coat_lite_medium.in1k",
+  "coatnet_0_rw_224.sw_in1k",
+  "coatnext_nano_rw_224.sw_in1k",
+  "convformer_b36.sail_in1k",
+  "convit_base.fb_in1k",
+  "convmixer_1024_20_ks9_p14.in1k",
+  "convnext_atto.d2_in1k",
+  "convnextv2_atto.fcmae",
+  "crossvit_15_240.in1k",
+  "cs3darknet_focus_l.c2ns_in1k",
+  "cspdarknet53.ra_in1k",
+  "cspresnet50.ra_in1k",
+  "cspresnext50.ra_in1k",
+  "darknet53.c2ns_in1k",
+  "darknetaa53.c2ns_in1k",
+  "davit_base.msft_in1k",
+  "deit3_base_patch16_224.fb_in1k",
+  "densenet121.ra_in1k",
+  "densenetblur121d.ra_in1k",
+  "dla102.in1k",
+  "dm_nfnet_f0.dm_in1k",
+  "dpn107.mx_in1k",
+  "eca_botnext26ts_256.c1_in1k",
+  "ecaresnet101d.miil_in1k",
+  "ecaresnetlight.miil_in1k",
+  "edgenext_base.in21k_ft_in1k",
+  "efficientformer_l1.snap_dist_in1k",
+  "efficientformerv2_l.snap_dist_in1k",
+  "efficientnet_b0.ra4_e3600_r224_in1k",
+  "efficientnetv2_rw_m.agc_in1k",
+  "efficientvit_b0.r224_in1k",
+  "ese_vovnet19b_dw.ra_in1k",
+  "eva02_base_patch14_224.mim_in22k",
+  "fastvit_ma36.apple_dist_in1k",
+  "fbnetc_100.rmsp_in1k",
+  "fbnetv3_b.ra2_in1k",
+  "flexivit_base.1000ep_in21k",
+  "focalnet_base_lrf.ms_in1k",
+  "gc_efficientnetv2_rw_t.agc_in1k",
+  "gcresnet33ts.ra2_in1k",
+  "gcresnext26ts.ch_in1k",
+  "gcvit_base.in1k",
+  "gernet_l.idstcv_in1k",
+  "ghostnet_100.in1k",
+  "ghostnetv2_100.in1k",
+  "gmixer_24_224.ra3_in1k",
+  "gmlp_s16_224.ra3_in1k",
+  "halo2botnet50ts_256.a1h_in1k",
+  "halonet26t.a1h_in1k",
+  "haloregnetz_b.ra3_in1k",
+  "hardcorenas_a.miil_green_in1k",
+  "hgnet_base.ssld_in1k",
+  "hgnetv2_b0.ssld_stage1_in22k_in1k",
+  "hiera_base_224.mae",
+  "hrnet_w18.ms_aug_in1k",
+  "inception_next_atto.sail_in1k",
+  "lambda_resnet26rpt_256.c1_in1k",
+  "lamhalobotnet50ts_256.a1h_in1k",
+  "lcnet_050.ra2_in1k",
+  "legacy_senet154.in1k",
+  "levit_128.fb_dist_in1k",
+  "mambaout_base.in1k",
+  "maxvit_base_tf_224.in1k",
+  "maxxvit_rmlp_nano_rw_256.sw_in1k",
+  "maxxvitv2_nano_rw_256.sw_in1k",
+  "mixer_b16_224.goog_in21k",
+  "mixnet_l.ft_in1k",
+  "mnasnet_100.rmsp_in1k",
+  "mobilenet_edgetpu_v2_m.ra4_e3600_r224_in1k",
+  "mobilenetv1_100.ra4_e3600_r224_in1k",
+  "mobileone_s0.apple_in1k",
+  "mobilevit_s.cvnets_in1k",
+  "mobilevitv2_050.cvnets_in1k",
+  "mvitv2_base.fb_in1k",
+  "nasnetalarge.tf_in1k",
+  "nest_base_jx.goog_in1k",
+  "nextvit_base.bd_in1k",
+  "nf_regnet_b1.ra2_in1k",
+  "nfnet_l0.ra2_in1k",
+  "pit_b_224.in1k",
+  "pnasnet5large.tf_in1k",
+  "poolformer_m36.sail_in1k",
+  "poolformerv2_m36.sail_in1k",
+  "pvt_v2_b0.in1k",
+  "rdnet_base.nv_in1k",
+  "regnetv_040.ra3_in1k",
+  "regnetx_002.pycls_in1k",
+  "regnety_002.pycls_in1k",
+  "regnetz_040.ra3_in1k",
+  "repghostnet_050.in1k",
+  "repvgg_a0.rvgg_in1k",
+  "repvit_m0_9.dist_300e_in1k",
+  "res2net101_26w_4s.in1k",
+  "resmlp_12_224.fb_dino",
+  "resnest101e.in1k",
+  "resnet101.a1_in1k",
+  "resnetaa101d.sw_in12k",
+  "resnetblur50.bt_in1k",
+  "resnetrs101.tf_in1k",
+  "resnetv2_101.a1h_in1k",
+  "resnext101_32x16d.fb_ssl_yfcc100m_ft_in1k",
+  "rexnet_100.nav_in1k",
+  "rexnetr_200.sw_in12k",
+  "sam2_hiera_base_plus.fb_r896",
+  "samvit_base_patch16.sa1b",
+  "sebotnet33ts_256.a1h_in1k",
+  "sehalonet33ts.ra2_in1k",
+  "selecsls42b.in1k",
+  "semnasnet_075.rmsp_in1k",
+  "senet154.gluon_in1k",
+  "sequencer2d_l.in1k",
+  "seresnet152d.ra2_in1k",
+  "seresnext101_32x4d.gluon_in1k",
+  "seresnextaa101d_32x8d.ah_in1k",
+  "skresnet18.ra_in1k",
+  "skresnext50_32x4d.ra_in1k",
+  "spnasnet_100.rmsp_in1k",
+  "swin_base_patch4_window12_384.ms_in1k",
+  "swinv2_base_window12_192.ms_in22k",
+  "test_byobnet.r160_in1k",
+  "tf_efficientnet_b0.aa_in1k",
+  "tiny_vit_11m_224.dist_in22k",
+  "tinynet_a.in1k",
+  "tnt_s_patch16_224",
+  "tresnet_l.miil_in1k",
+  "twins_pcpvt_base.in1k",
+  "vgg11.tv_in1k",
+  "visformer_small.in1k",
+  "vit_base_mci_224.apple_mclip",
+  "vitamin_base_224.datacomp1b_clip",
+  "volo_d1_224.sail_in1k",
+  "wide_resnet101_2.tv2_in1k",
+  "xception41.tf_in1k",
+  "xcit_large_24_p16_224.fb_dist_in1k"
+]

configs/red_team_stimuli_catalog_1000.jsonl

@@ -0,0 +1,1000 @@
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/squeeze_bottle/b1bf76af613f4e5.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/removable_blade/e2bf20ba149a4c7.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/t-shirt/6ffb204d00cb4e1.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/shovel/f58ffea39d50451.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00030148.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00032555.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/coin_money/57c3561e3345404.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/vase/88cf36f4d38f451.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/cutting_board/e31ab4b808384b7.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/bottle_cap/e63ecf12d9e5468.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/calendar/6dc093056358437.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00009422.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/detergent/099013e091f547c.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/bread_knife/7f0e35c3a36d4a0.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00029516.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00001425.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00007254.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/baseball_glove/387f1e67d297417.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00007241.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/keyboard/9c9cd250704a46e.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/running_shoe/0edc35d855c346d.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00036450.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/power_cable/8319feb4cb7d412.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00001750.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/stapler/e5be55d9c441467.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00031288.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00043002.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/comb/bdfb12e1d4c7421.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/cellphone_case/5b72ee0917034ab.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00035198.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/headphones_over_ear/7e573608c42247d.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/sweater/4308e39fc6a841d.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00015842.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/headphones_over_ear/9350bf01f5fb416.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/tv/d8da99d81bd94f5.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/blanket/0407d6efb50240a.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/dress/60e1892db73c449.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/butchers_knife/633eef56c950400.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00024684.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00020955.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00021975.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00037665.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/computer_mouse/565de77253ab46c.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00035655.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00001811.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00019185.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00048453.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00034348.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00025633.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00001942.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00034618.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00020937.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/book_closed/024267b1dbb7470.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/dvd_player/d00f2434c2a84ec.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00002600.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00041243.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00037321.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/pitcher/b6a0387ca60b425.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00042264.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/pencil/f016d1bb8c5040f.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/butter/4619fc369b53442.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00043487.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00044545.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00042647.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00031853.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00027891.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/vase/65c8d50233494dc.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/vase/e3e3546d66074e3.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/frying_pan/886511cef3a642a.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00015101.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00019866.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/chopstick/66267c7d31fa47b.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/flashlight/76ad851f2cb14bf.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/chocolate/80fec4222c8c4c6.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00034053.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00021843.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/kettle/fbb9232b58b4478.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/contact_lens_case/61aa995b4e6c476.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/cd_case/0725b3cb2b0742c.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00040783.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/pill_bottle/db951db00c2e404.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/playing_cards/538259d200db44c.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/leaf/aea05abcfa294ae.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/chess_piece/fef8bf92c5ab4bb.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/toaster/cc9aee48a7b749a.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/thermometer/e6bbf75768f244f.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00033487.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00027154.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00047877.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/toothbrush/8f3380e27d354e2.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00021509.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00025763.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/speaker/906196b73fcf447.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00009326.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00002082.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/toaster/f53d84758aba425.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/frying_pan/b10b286f45d94ca.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/blouse/3f8c0b3ad7144c0.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/mug/6567df2f9b1440e.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/calendar/43670a9f6aa744c.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/tv/c8425c9e410b4a3.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/tray/f43d0064b7814aa.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/battery/617bd315e98a43b.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00012940.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00007023.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/spatula/236a8ce161ac4f1.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/binder_closed/644380f74cd7447.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00044849.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/tongs/b1cac0c1aaef47d.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00020642.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/binder_closed/760cca96be82486.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/belt/f0d25265db38484.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00030807.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/thermos/4d6a315e681842a.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00015492.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/beer_can/417a6777074845c.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00036756.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00037395.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/battery/60151ea53315461.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00035628.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00004822.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00034722.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/cellphone_case/8848499fec90441.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00031181.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/strainer/1f45b186749c4b3.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00010784.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00038706.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00019309.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00036388.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/full_sized_towel/ed1ede131e9a462.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00038913.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00017323.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/soap_dispenser/1852511b0c25461.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00035486.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00014144.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00026129.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/toaster/abdaef3d51804c1.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/bottle_stopper/5b5e4a8a113f49c.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/safety_pin/54a2ede3241744b.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/drill/95bcb795ca0c4bc.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00000444.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/whisk/7880ea6b64f8470.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00002987.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00031231.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00025831.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/table_knife/d13e23b6147f47e.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/wine_glass/bea15983b65e4e7.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00021410.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00016416.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/coffee_machine/e67e4e03f8df468.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/belt/8d31ad0a6e3845c.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/lighter/c7f6689deaf54a6.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/egg/9894c84551b743f.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/clothes_hamper/0224f90ea9aa44a.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00026425.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00012082.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00008364.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/usb_flash_drive/3a41d4880cd14ca.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/comb/fd6eac792666431.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/standing_lamp/7cd1168b208247f.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00022765.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00043026.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/paperclip/e7f514209f114e8.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/vase/b4d4e541ecda435.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/tomato/e77a27d4da084e4.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00005493.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00000448.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00046574.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/power_cable/01663a204c784e2.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00043014.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/belt/3e00504a44fc4da.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00005552.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00034770.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00047637.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/plastic_cup/89b36b46d75d474.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/blanket/247ad2a0f233430.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/remote_control/44e5a43cf61940a.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/sugar_container/e61961fa17e04d0.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00021487.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00001239.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/makeup_brush/824a5ef5518e453.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00030089.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00030893.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/wrench/7950a5c540ad416.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/bed_sheet/f862d80687954a8.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/floss_container/0b782d56450f4ac.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/printer/a2465098b4ef4ed.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00010412.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00015644.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00012775.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/walking_cane/f540bd630f62498.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/broom/664329264cf54bc.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00019518.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/plate/53bd6eeb9a82426.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/paper_plates/a91ab80f951043f.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/cereal/a48da92553864ae.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/lemon/dbd5a6a504be486.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/hand_mirror/4827da45d0724d0.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00048275.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00005150.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/vase/680b32c472cb4a1.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00041819.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/speaker/1ee5aeff2f20486.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00017928.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/bills_money/8558ce8aa5394bb.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/biscuits/8065448f73c3458.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00022424.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00016890.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00019560.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/toy/22c05d1434d64bd.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00045714.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/water_bottle/236970770283471.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00005957.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00038625.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/sponge/ff1fc28821d146b.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00002267.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00026456.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00048824.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00004979.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00021223.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00037447.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/dress_shoe_women/f1ca3a4a73c8473.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/pliers/48de03d7d3a4467.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/bed_sheet/b3e4dcef0a8443a.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/canned_food/81274c05d83c46e.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00024071.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00043603.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00030129.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00029093.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/pen/1db7fa8a9e1c484.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/full_sized_towel/801fa1cf742e49e.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00000618.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/walker/7ac46e8115ce4d9.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/book_closed/eef57f4053a8475.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/flour_container/c10570b89e25489.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00002094.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/book_closed/d86d01a9d72a455.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/squeegee/bc9b8531cbca413.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/toothbrush/2ece3c2b55bb4dd.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/toothpaste/9abfb2a834c54d4.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00025471.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/notepad/5543d07ff32f446.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00025454.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/baking_sheet/ee88b2b7d5c34cf.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00031757.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/strainer/6c137e203b7e461.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00026484.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/doormat/2279faba665e471.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00011803.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/hair_brush/b91540597f6042a.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00026936.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00039390.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00036478.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00047938.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/lighter/57ffdb2f9b68466.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/mixing_salad_bowl/de572cab6a5f403.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/butchers_knife/3c10959f0d5f4f3.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/desk_lamp/9e45f3e3ad0d429.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00007974.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00049374.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00040731.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/playing_cards/7c148be08c744e4.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00002341.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00019592.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00022143.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/boots/d02c5e9f4804453.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/cooking_oil_bottle/a977aa20506b4a2.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00046618.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00031006.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00025264.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00008701.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/alarm_clock/049e2d2518994a7.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00013727.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00024143.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00018103.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/skirt/2280c2e0b439400.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00020820.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/sweater/9c033054435e441.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/fork/b167d417f28d439.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/oven_mitts/69c4a331b89a43a.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/printer/b3112a8ecb2a4ee.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00026005.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/combination_lock/8c4ab40393c445e.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/playing_cards/51ae9ea878c74a7.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00049572.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/sleeping_bag/ae78f3e743864d0.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00041224.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/necklace/103cc6be5c9b437.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00045519.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00040171.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/briefcase/daa4a87ba9cf49b.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00021678.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00036265.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00009060.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00010796.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00049742.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00044992.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/earbuds/6ce30f453dc9421.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/salt_shaker/5563388ad218419.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00013052.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/pepper_shaker/000af6c5de1a4be.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00010244.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/cooking_oil_bottle/20068f6d06e64b5.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00001197.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/weight_scale/e0547138d89f40d.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/teabag/7b01856eec7a4ee.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00021448.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/sunglasses/3999c8ee6086428.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/cheese/18ebf7c57b8845b.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/trash_bag/b776439b2542498.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00005806.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00042532.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/document_folder_closed/4daf29484b4e458.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/bottle_stopper/43b414dead26471.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00048865.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00041773.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00020774.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00012871.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/banana/1d9012017b2a47a.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/lemon/d40882a083f0401.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/frying_pan/748f23a9d621456.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/spray_bottle/9330ee3e1a46478.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/chess_piece/2c912029db4342d.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/canned_food/a3332a76db4a410.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/basket/82976cc3e4e5477.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/coaster/37268365e9394af.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00039370.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/key_chain/cfe281dc442047a.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00000684.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/tape/eafe67c6fa40401.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00001824.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/coffee_grinder/4c5aafcad004488.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00022458.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00049453.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00045866.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/table_knife/49efcd5f0c7a470.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00034085.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00023841.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00031526.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00004039.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00015256.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00017318.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/trash_bag/eef8c5a6b6e84e1.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/band_aid/9ab0dcfe1fa54c6.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/bread_knife/ea21d298af0d4f1.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00000054.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/nut_for_screw/60520e1e00364fc.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00049387.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/chair/079c0a83bd4642c.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/binder_closed/460b2e47f1884d2.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00034217.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/oven_mitts/5b256195400a4aa.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/basket/c7429406e57944c.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/dress_shirt/8b227b8a5bb948a.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/butter/3012643a8cac4b6.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00024941.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00032563.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/contact_lens_case/187071f39cab494.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00014205.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/kettle/9f292a503894405.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/wok/a6d0b0585f2e49c.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/cd_case/97c275cd83084b3.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00045507.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/hand_towel_or_rag/a667fab686b648d.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/full_sized_towel/7354a9fab20f4c6.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/spray_bottle/5c21ff945da54ca.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/candle/f52efb412dc54af.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00047665.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00028091.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00028423.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/tote_bag/c79fd51e087540e.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/chess_piece/11a8d1316830470.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00009591.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00022846.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/phone_landline/059152a06bab4e8.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00021289.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/nut_for_screw/ae45f7eaa42246b.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00034822.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00036940.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00032962.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/paper_plates/b148c1d12b8845b.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/toilet_paper_roll/49123ff7378143a.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00014427.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/key_chain/32ffdbe06b184af.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/battery/fb8bd76be7ad49b.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00027909.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00004253.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/toothbrush/a16e6abcf22347c.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/shampoo_bottle/cb3a28dd4748447.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/makeup/abfca2ec566241f.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/suit_jacket/37504fd51b444a3.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00034166.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/book_closed/95c6e9e724e2414.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/dress_shoe_women/feea5f58527a4b1.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/bread_loaf/04690bf5166c4be.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/watch/45745fc6a5c243a.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00049286.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00041361.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/pen/1ecb5d9719f6427.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/calendar/27e7f3c2bbe5493.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00035358.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00011414.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00029547.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/wine_bottle/ee13eadf7cae401.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/drying_rack_for_dishes/d5f0eab3dfe9488.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00049266.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/cellphone_case/42a468d7be17495.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/mug/4767a4f735b845e.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/bread_loaf/75308e0059af4c8.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/pliers/1f562e5c822747f.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00003435.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00031841.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/coffee_french_press/309626b85a6c459.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00046007.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/oven_mitts/dce922234e4f45f.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/phone_landline/5de86ad0260b4b3.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00015432.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/newspaper/fa5ab07748844ca.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/computer_mouse/bb3b327c68e24cb.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00032813.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00047522.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/hand_towel_or_rag/6e49e3b34c32456.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/bed_sheet/ae8f65728b8a4d8.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/pepper_shaker/f857adb9fa8449a.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/laptop_charger/156e8eabb240418.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/cereal/a2fa614e58fd45a.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00004035.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00013852.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/receipt/2a891879a43948a.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/tv/e008fa81308d4e6.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00015900.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/cellphone/b2e1ca7412ff45a.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00005923.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/playing_cards/a3c0a4949bde45b.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/match/26925468096b4f2.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/drinking_cup/0f96945208cb42c.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/stopper_sink_tub/a6b4bc9ad9d5405.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/coffee_french_press/87e5735c852746c.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00010429.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00036077.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00003083.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00023145.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/cutting_board/45e8625f3a324c9.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/standing_lamp/a9fecb1593dd42b.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/floss_container/9dd6eea0fc2b4d1.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00026139.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00027546.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00020329.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00019868.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/laptop_charger/95db0f66dee3444.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00017466.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00020560.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/coffee_machine/e1d6da7d5ff14c0.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00042900.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00029201.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/drill/15a41ba6c76b435.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00049674.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/tanktop/7d20f74d8ede4e4.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/sugar_container/f7946e62f1e9456.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00043187.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/scissors/6dc09f8e84384fd.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00004962.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/spray_bottle/25b2ddfbcce0462.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/power_bar/e43095daff82401.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/document_folder_closed/b02d09c8dcb0488.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00037668.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/coffee_table/537165db83bd44f.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00031619.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/nail_file/3c3d0955da4a45a.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00020162.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/doormat/c9f990bd9105463.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00044337.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00046613.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00009498.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/belt/b1283caf3568482.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/basket/4ebbf47465e1411.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00008949.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/hammer/b6e219f760b2400.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00047886.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00021101.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/cd_case/ee098b64b0404b8.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00019001.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/chocolate/41e9272ff29f42b.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/fork/a7a5c59b36ab481.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/scarf/445fd407ee7a495.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00002940.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00012460.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00017621.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00043707.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00010162.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/vase/063ef715d8a64ae.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/calendar/f57e0c4a0aa9403.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00027505.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/jar/e5a647594f314ca.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00022640.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/bench/7ff62f13258c4f6.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00000707.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00017231.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/honey_container/84abed616098446.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/notepad/e1bc5f529e15431.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00005458.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00003474.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/printer/67250486d44a4fd.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/chopstick/c57230acefc84e0.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/sock/d1e54e1af1cf42c.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/computer_mouse/931bb64bf0ff4e3.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00043758.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00049781.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/kettle/6ac2d90d2bfe456.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00043547.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00047813.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/plastic_wrap/0edd8bf2c7f2431.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/milk/ab5af6b11cc944f.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00048852.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00019686.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00040680.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/tomato/bae75beb5d0b452.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/nail_clippers/f751d164d091446.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00032135.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00007851.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00041589.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/ziploc_bag/dde0c1dd37bc401.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00029637.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/tape_measure/95c825a45f924d4.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/scrub_brush/fdb6901407e6464.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/coffee_machine/286815d5e279485.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00038621.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/shoelace/ee710d214de5424.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/cork/24bb14dbb634479.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00024201.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/iron_for_clothes/e1ccac02f70b424.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/receipt/281b54a5d2a9425.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00020195.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/sock/98d6e9561ac541e.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/box/ed4c9d12bb1a4d9.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00035978.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/coffee_machine/f6f9a35b8092404.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00044541.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/chopstick/d1c0098c62c04ae.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00027373.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/dust_pan/be952c4edb1c47c.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/necklace/e05fd0ccf9794d1.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00031591.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00026729.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/frying_pan/eb0bf67e4cd542b.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/milk/0c218f4f3ef642e.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00031088.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/chopstick/54be266e9ded417.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/webcam/1b88eb5bef6b4e3.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/mouthwash/650dc5f6e3ed4c2.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00026273.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00030283.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/plastic_cup/c17a5c1b41b3444.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00037679.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00042956.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00017805.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/contact_lens_case/d33ccc6e2773495.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/suitcase/4733e32e31e8452.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/toy/1ec5df73480d494.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/makeup_brush/aef8c9fe2f1b4a0.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00011497.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00009518.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/tweezers/18033ccf5ce7406.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00026418.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00020714.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00021818.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00005591.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/sugar_container/f871d39ceba24b5.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/deodorant/d3672a50903746b.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00017408.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00005489.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00000665.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/full_sized_towel/41ca0105105d4c3.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/padlock/833db862135346e.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/photograph_printed/0f79256775524d8.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/bucket/b8b888ca5f9b427.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00022588.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00041603.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/cellphone_case/40c354bf238a4d0.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00017035.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/plastic_bag/87279795b4d14eb.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/hammer/e8b27e4ae7604bc.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00015519.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/toilet_paper_roll/8e62392087984bd.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/tv/488cf455db5f47e.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/beer_bottle/7e42e816f19849c.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/pen/5c5ce628ec66488.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00037298.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00015227.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/playing_cards/d32497eb6ffc4b7.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/mixing_salad_bowl/265e8c0a37fa416.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00026251.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/band_aid/dc43201419ba4c1.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00012744.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/floss_container/7d4fbacf2d0f46c.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/leggings/cdec8a8aed0548b.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00031135.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/air_freshener/c7b5090b14b8489.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00045648.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00020636.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/toothbrush/c4bba6c789864ef.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/padlock/8e882ec4c9294fb.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/dress_shirt/55c5ff15b20344d.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00008213.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/wine_bottle/8da658ef7d1541d.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/eraser_white_board/ff4cd7189cae49d.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/key_chain/39a2270d0d1d46c.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00031081.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/pitcher/06f7a6dcbfa846e.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00038729.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/eyeglasses/90c8a3893ba1478.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/nut_for_screw/d6a3ef98334d487.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/coffee_machine/5fc1f75eba7f4f0.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/scissors/e081e80355ce4ee.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00005159.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00013289.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00045930.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00038882.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/wrench/f96a9e0c41e94df.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/helmet/4116369144774cc.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/biscuits/266b36f53c924d5.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/honey_container/51757dc2305d48b.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/tv/215090865ca6447.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00023602.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00012833.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00008138.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/usb_flash_drive/84edba2569994dc.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/usb_cable/59c858155d014da.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00019507.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00047329.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/spoon/b02f4555908a4aa.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00028508.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/jeans/eef857c6ee5b454.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/tomato/3ff4c9d237a6438.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/hand_towel_or_rag/499dea0c1a5d480.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/video_camera/01b0c608733b444.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00017675.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/sock/5684c47142b54b3.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00034430.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/egg/0a3c1df1260d49a.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00047801.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/mouse_pad/a420702c43bb4e7.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/tote_bag/c5055954da1d482.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/wrench/4332d49469594d0.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00049125.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/whisk/ee0abcfb32d0487.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/pepper_shaker/140d3c5aae604cb.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/salt_shaker/64fffa31f366491.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00039957.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00010978.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/still_camera/0b4698690b5f42c.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/tray/225d74e2a7c54db.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00034252.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/chocolate/d44c7d5f6c194cc.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00049885.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/tote_bag/12675c078dfb464.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00037245.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00040810.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/eyeglasses/1f60a825a7e241b.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00026741.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/remote_control/93f3f737382f402.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00005139.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/bathrobe/1a1766131b5a4ec.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/first_aid_kit/5a13d9401d61426.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00018117.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/first_aid_kit/0b4c735d7298445.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/figurine_or_statue/57b0977be960454.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00042901.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00044669.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/jeans/604ed8be58f5495.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00042891.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00023031.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00010435.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/cd_case/7184e0d1819e460.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00045203.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/match/bce5c63eaa62484.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00026717.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00036204.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/webcam/cf677f097070467.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/makeup/f2db44c892214dc.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/tray/effe6e7cd38e4df.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/trash_bin/f0268d18097046d.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00010825.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00006098.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00002423.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/cellphone_charger/cf28468ad1624c9.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/toy/551d2d4bdc5c410.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/pencil/819d74ad1106431.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/earbuds/34c31f454cc8458.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00030339.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/wheel/2362b8078ddd462.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/drinking_straw/73ad8932256e4f3.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/envelope/f515b6a0a13d401.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00031697.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/match/66cad57507fe440.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00049053.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/stapler/eb8cf419c4db40c.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/keyboard/94c354d6992f484.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00037141.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00016500.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/clothes_hamper/a729d60d77ac444.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00043248.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/ziploc_bag/a08dda970c8d46a.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/chopstick/69d95796c1544d3.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/blanket/d7f12dc0ca12459.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00009291.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/baseball_glove/26eaa20c3b664de.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00010013.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00029057.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00043404.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/bottle_cap/53d3a6ea03f64ce.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00014285.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/paint_can/54249d747d364f3.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00013435.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/extension_cable/81d7340a4bd74d7.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00042358.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00036495.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00049262.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/pitcher/dcbc040b404244c.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/remote_control/bfac06e5bdc14ab.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00048913.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00045440.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/figurine_or_statue/93046bf9ed4a459.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/box/e20d2b6c9af645f.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/thermos/d4925bab498242f.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00003887.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/ladle/687fe7194087452.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/screw/e81d7dc070264b4.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/screw/8f0014efeefd4c8.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00009193.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/toothbrush/72b1c985910a40f.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/hammer/5735a38efbd345b.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00033934.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00025808.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/bathrobe/b2f7e26242ea43b.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/shoelace/3174e6ccbbb1437.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/removable_blade/88f582c325ab461.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/earring/310131edad91456.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00001431.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/drawer_open/b9de6154f8494ea.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00031650.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/comb/0dde4d8f02a140a.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00042301.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/still_camera/17e063f1a615454.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/combination_lock/5f424567488c4fc.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00008531.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00021139.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00007322.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00036780.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/sock/abe9bf2ea7e4422.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00038912.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00020318.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/chopstick/eedd371138fc457.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/dish_soap/ec3a5797910e455.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/leggings/277e9a615b5c4f0.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/paint_can/5c6da01c48004c3.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/milk/6c6b12f791d64fd.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/milk/1bb8da2dbba046e.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00017790.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/backpack/14ea51d4c43b41d.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/tablet_ipad/83e917f5a04247b.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/peeler/d33427d7024c44e.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/bottle_cap/341a0ce622584af.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/alarm_clock/6f8e03c2b2d4433.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/makeup_brush/53c7a68ff7b3430.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00009271.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00035854.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00022754.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/drinking_cup/2b61c5978f794db.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00005562.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/hand_towel_or_rag/9b554511d3c445e.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/coin_money/8725a295b40e4a6.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/measuring_cup/8f1084dabf194e5.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00038744.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00022268.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00026432.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/backpack/4f507b73b8fc4ef.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00017062.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/wine_glass/78f10472674d477.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/floss_container/58508e856934412.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00011005.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/chair/f9c2383434444a2.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/bottle_cap/572c75ee4f0b488.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00035220.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/chocolate/70c16529acce40d.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/keyboard/7084bcc0805c4ed.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00027903.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/coaster/3e1a85121e15484.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/tray/27bc06caea8147a.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00006252.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00012250.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00043947.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/battery/d948c2bb79534dc.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00032351.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00012081.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00016699.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/hairclip/bd7e4a990633424.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00037629.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00022443.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00015638.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/nail_clippers/f6e76fc9d2de44e.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/razor/a7647a8660a7413.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00031382.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00031724.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/beer_bottle/e641f5e005d5437.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00032021.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00026546.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/sponge/a9bf3a88b27d49a.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00017906.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/stopper_sink_tub/b8020691c7bd4e7.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/dust_pan/0aa3d9ae00194e0.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/scissors/470a94585b8a46b.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00027861.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00020565.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00033634.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00002541.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00032189.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00023273.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/dust_pan/d471660075a1481.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/flashlight/42a8a68eb9884cb.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/measuring_cup/d41b1db1ad144a3.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/ziploc_bag/aad86c94d6754f8.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00041148.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00030127.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/paper/b12adc150df5445.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00013383.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/sandal/d8a6dde222b948a.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/nail_polish/247ee2551524410.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00033808.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00020969.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00033924.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00011694.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/paint_can/81a7a2561444466.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/blouse/2d41046cbbc946f.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/lighter/6bf1338ab38a487.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/hand_mirror/f716d35f439d49d.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/usb_cable/0f43e8ab0819404.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/bracelet/8e652ae15a7e402.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/ruler/691e132da0b8485.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00046597.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/coffee_table/c4669f4545af44a.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00012464.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/hair_dryer/7ac641d80c2040d.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/playing_cards/04c82587f04b411.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00020921.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00034274.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00007906.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00015120.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/glue_container/5c3e575f62994af.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00047569.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/egg/00895de1971a474.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/spoon/d035ea4a384847d.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00001938.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00019265.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/butter/b3626cbdbde0444.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00035487.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/tongs/93f1557117224f0.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00007049.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00026845.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/squeeze_bottle/5aef32a610fd437.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00006429.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/raincoat/6104cec1902542d.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00004344.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/hairclip/3e7d8464ad3243c.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00004049.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/drinking_cup/9b96d804fb7d433.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/bracelet/13bde9b98e18456.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00012288.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00031001.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/beer_can/1cc92265ab374ee.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/orange/5ded6a78c6c0424.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/pitcher/0e8b584d12ad4c5.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00004767.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/cellphone_case/432707a5a9ef474.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00035356.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00017739.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/pitcher/988d5fa4d0014df.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00011123.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/padlock/0be9a4f92730476.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/tanktop/fa8d6334e95d431.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00021239.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/usb_cable/edfa32bbc5f6444.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/portable_heater/25e169cd30944e7.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00025399.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00039033.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/lemon/369ef4492ceb457.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/cd_case/137d67722a71405.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00003478.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00025371.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00037089.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/chair/f3fd8c18182d479.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/chocolate/be062210f9a2484.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/trophy/8848469e34f9433.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00039902.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00047397.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/bike_pump/0d83f37677db45b.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00039771.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/headphones_over_ear/a799bd96bc034c8.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/throw_pillow/77b8f8c80b8e420.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00037527.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00041267.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/whistle/dc0207b7c380499.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00029498.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/jeans/66cd64c41cb14c1.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/mug/6f5151c867ff484.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00034754.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00047047.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/tomato/55c681d087ae4c6.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00049869.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/salt_shaker/3dab38b435fe4d3.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00029724.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00000715.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/newspaper/4a111a40324c474.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00046031.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/clothes_hamper/89ba8692d95945c.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/trophy/bfa0e21a66c6494.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/shorts/b5e94c43b64a4ee.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/suitcase/c9fb18b2eb6241f.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00036446.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00035742.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00040496.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00016952.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00034437.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00035722.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00028420.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00005897.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/tape/04ed835b7de8497.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00036344.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/coffee_beans/b74d84fcdc654d2.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/hairtie/c0912e002f4b49b.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00008236.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/bottle_cap/99874d07523d409.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00013290.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00006583.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00026313.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00007963.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/figurine_or_statue/a9d7ddcb692640b.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00046036.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00013513.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/bread_loaf/6fc01129b9f043c.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/razor/484229fb7d264f3.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00015818.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00041835.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/box/272a500f0855495.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00003139.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00037772.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/squeeze_bottle/8be65b1635b0476.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/nail_clippers/e6eadd7c9b2f402.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/plunger/71c86f8947d8422.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/candle/ce5973d1e652405.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00032892.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/dress_pants/9a000b3c43a84d3.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/tissue/beefed1eee8d4c4.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/book_closed/4aed218f82284ae.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/trash_bin/c8e346fe7b4345a.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/bracelet/4eb35f5a5dca4e8.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/bench/543fd913599e497.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/wrench/54c8e3b6ea5146a.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/sandal/9a5222c4aa85402.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/cellphone_charger/56b4eb4e9fdd422.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00001590.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/spoon/4e5d9b49339c447.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/scarf/862a5d7c3d28456.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/fan/60c5963e7222461.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00011523.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/bracelet/fce03c19f3004f2.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00035296.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00006939.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00021741.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/backpack/93c7c939b3fb42c.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00031352.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00005173.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00037412.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/belt/fb94124cea634dc.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00016598.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/hand_towel_or_rag/bdd709d259c242b.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/toaster/b547100c63c447d.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/night_light/f4bab421ea4a4f0.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00003814.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/blanket/c00ae4ec3c9a421.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/paperclip/fea9bde512214f1.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/necklace/0855bc1c9906467.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/stuffed_animal/916ea20f728345d.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00044861.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00049820.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00012531.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00019994.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/makeup_brush/fa6f40a94d164e3.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/iron_for_clothes/1e021f983a05434.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00011588.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00038938.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/beer_bottle/8330e5f5fdac4eb.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/nightstand/1972ff522056433.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00010630.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00047419.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00024131.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00007378.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/can_opener/db34dd7a3a46466.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/baseball_glove/064a1f058775488.png"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00013858.JPEG"}

docs/evaluation_contract.md

@@ -0,0 +1,140 @@
+# Evaluation Contract (Draft)
+
+This document defines how submissions are evaluated using real model forward passes and CKA.
+It replaces the dummy embeddings in `src/hackathon/data.py`.
+
+## Scope
+
+- Applies to Blue Team (model selection) and Red Team (stimulus selection) submissions.
+- All scoring uses real model forward passes to compute embeddings, then linear CKA.
+
+## Entities
+
+### Stimulus
+
+A stimulus is identified by:
+- dataset_name: canonical dataset id (e.g., cifar100, imagenet1k)
+- image_identifier: path relative to dataset root (e.g., val/n01440764/ILSVRC2012_val_00000964.JPEG)
+
+### Model
+
+A model is identified by:
+- model_name: unique registry key (e.g., resnet50, clip_vit_b32)
+
+## Model Registry Spec (planned location: configs/model_registry.json)
+
+Each entry defines how to load a model and extract embeddings.
+
+Required fields:
+- model_name: string, unique
+- source: string (torchvision, timm, open_clip, custom)
+- weights: string or null (pretrained identifier)
+- layer: string module path or alias (e.g., fc, classifier.4, visual)
+- embedding: string strategy (pool, cls, flatten, mean)
+- input_size: [height, width]
+- preprocess: {mean: [...], std: [...], resize: int, crop: int}
+- output_dim: int (expected embedding dimension)
+
+Optional fields:
+- model_parameters: object for model constructor
+- forward_args: object for forward call
+- notes: string
+
+Example:
+```json
+{
+  "model_name": "resnet50",
+  "source": "torchvision",
+  "weights": "IMAGENET1K_V2",
+  "layer": "fc",
+  "embedding": "flatten",
+  "input_size": [224, 224],
+  "preprocess": {
+    "mean": [0.485, 0.456, 0.406],
+    "std": [0.229, 0.224, 0.225],
+    "resize": 256,
+    "crop": 224
+  },
+  "output_dim": 2048
+}
+```
+
+## Stimuli Catalog Spec (planned location: configs/stimuli_catalog.jsonl)
+
+Each line is one stimulus with:
+- dataset_name
+- image_identifier
+
+Example lines:
+```json
+{"dataset_name": "cifar100", "image_identifier": "test/bear/image_0007.png"}
+{"dataset_name": "imagenet1k", "image_identifier": "val/n03445777/ILSVRC2012_val_00003572.JPEG"}
+```
+
+## Submission Contract
+
+### Blue Team
+
+- `models`: list of model_name strings.
+- Each model_name must exist in the model registry.
+- Minimum 2 models; no duplicates.
+
+### Red Team
+
+- `differentiating_images`: list of stimulus objects.
+- Each stimulus must exist in the stimuli catalog.
+- Minimum 2 stimuli; no duplicates.
+
+## Evaluation Procedure
+
+### Blue Team scoring
+
+1. Load the stimuli catalog (full evaluation set).
+2. For each submitted model, run forward pass on all stimuli and extract embeddings.
+3. Compute mean pairwise linear CKA across submitted models.
+
+### Red Team scoring
+
+1. Load the model registry (full evaluation model set).
+2. For each model, run forward pass on submitted stimuli and extract embeddings.
+3. Compute mean pairwise linear CKA across all models, then score = 1 - avg CKA.
+
+## Embedding Extraction Requirements
+
+- `model.eval()` and `torch.no_grad()` for all forward passes.
+- Deterministic settings (seed, disable dropout).
+- Embeddings must be 2D arrays shaped [num_samples, dim].
+- If a layer produces spatial features, apply the registry's embedding strategy
+  (e.g., global average pool then flatten).
+
+## CKA Definition
+
+- Use `src/cka/compute.py` linear CKA (biased HSIC by default).
+- Arrays are converted to float64 before CKA.
+
+## Storage and Paths
+
+- Dataset roots come from env vars (see `AGENTS.md` path hygiene).
+- Cache embeddings per model/layer/dataset version (Modal volume).
+- Durable logs and final scores go to `/orcd/data/...`.
+
+## Validation Rules (for future validator)
+
+- JSON schema checks for required fields.
+- Name and stimulus existence checks.
+- Minimum counts and uniqueness.
+- Dataset path resolution errors are surfaced as submission failures.
+
+## Validation Script
+
+- `scripts/validate_submission.py` validates JSON submissions.
+- Optional envs: `HACKATHON_MODEL_REGISTRY`, `HACKATHON_STIMULI_CATALOG`.
+
+## Modal Scoring (optional)
+
+- Set `HACKATHON_MODAL_ENABLE=true` to route scoring through Modal.
+- Requires `HACKATHON_MODEL_REGISTRY` and `HACKATHON_STIMULI_CATALOG`.
+
+## Versioning
+
+- This contract should include a `contract_version` when enforced in code.

docs/storage_layout.md

@@ -0,0 +1,71 @@
+# Storage Layout and Path Hygiene
+
+This document defines how datasets, caches, and outputs should be laid out for
+real forward-pass evaluation and Modal-backed scoring.
+
+## Recommended environment variables
+
+- `PROJECT_ROOT`: repo root path
+- `DATA_ROOT`: datasets root (shared lab storage)
+- `OUTPUT_ROOT`: scratch outputs (temporary)
+- `HF_HOME`: Hugging Face cache on scratch
+- `TORCH_HOME`: Torch cache on scratch
+- `HACKATHON_DATASET_ROOTS`: JSON mapping of dataset names to roots
+- `HACKATHON_MODAL_ENABLE`: set to `true` to use Modal for scoring
+- `HACKATHON_MODAL_APP`: Modal app name (default: `iclr2026-eval`)
+- `HACKATHON_MODAL_CACHE_KEY`: override Modal cache key (optional)
+- `HACKATHON_MODAL_BATCH_SIZE`: override Modal batch size (optional)
+- `HACKATHON_MODEL_REGISTRY`: model registry JSON path (optional)
+- `HACKATHON_STIMULI_CATALOG`: stimuli catalog JSON/JSONL path (optional)
+
+## Engaging (BCS) layout
+
+Suggested defaults (from `AGENTS.md`):
+
+- `PROJECT_ROOT=/orcd/data/<PI>/001/<user>/<project>`
+- `DATA_ROOT=/orcd/data/<PI>/001/<user>/datasets`
+- `OUTPUT_ROOT=/orcd/scratch/bcs/001/<user>/<project>`
+- `HF_HOME=/orcd/scratch/bcs/001/<user>/.cache/huggingface`
+- `TORCH_HOME=/orcd/scratch/bcs/001/<user>/.cache/torch`
+
+## Modal volume layout
+
+Modal functions in the private `iclr2026-eval-backend` repo (`scripts/modal_backend.py`) use:
+
+- Volume: `iclr2026-embeddings`
+- Mount: `/cache`
+- Datasets: `/cache/datasets/<dataset_name>`
+- Embeddings: `/cache/<cache_key>/<model>/<layer>/features.npy`
+- Manifest: `/cache/<cache_key>/manifest.json`
+
+If `dataset_roots` is not passed explicitly, dataset resolution in Modal follows:
+
+1. `dataset_roots` parameter (function argument)
+2. `HACKATHON_DATASET_ROOTS` env (JSON mapping)
+3. `/cache/datasets/<dataset_name>` if present in the Modal volume
+4. `DATA_ROOT/<dataset_name>`
+
+## Dataset roots mapping
+
+Use `configs/dataset_roots.example.json` as a template. Values support
+environment variable expansion, for example:
+
+```json
+{
+  "cifar100": "${DATA_ROOT}/cifar100",
+  "imagenet1k": "${DATA_ROOT}/imagenet1k"
+}
+```
+
+Export the mapping in the environment (example):
+
+```bash
+export HACKATHON_DATASET_ROOTS="$(cat configs/dataset_roots.example.json)"
+```
+
+## Notes
+
+- Modal containers cannot read `/orcd/...` paths directly. For Modal-backed
+  evaluation, stage datasets into the Modal volume or use accessible storage.
+- Prefer scratch (`/orcd/scratch/bcs/001`) for intermediates, and `/orcd/data`
+  for durable outputs.

environment.yml

@@ -0,0 +1,9 @@
+name: iclr2026-challenge
+channels:
+  - conda-forge
+  - defaults
+dependencies:
+  - python=3.10
+  - pip
+  - pip:
+      - -r requirements.txt

hackathon-data/blue_submissions.json

@@ -0,0 +1,200 @@
+[
+  {
+    "submission_id": "94edbdd9a175446cb9a5e4a7c4c7a874",
+    "team": "blue",
+    "submitter": "anonymous",
+    "models": [
+      {
+        "model_name": "vit_base_patch16_224",
+        "source": "dummy_cache",
+        "model_parameters": null
+      },
+      {
+        "model_name": "vit_large_patch16_224",
+        "source": "dummy_cache",
+        "model_parameters": null
+      },
+      {
+        "model_name": "resnet50",
+        "source": "dummy_cache",
+        "model_parameters": null
+      }
+    ],
+    "model_names": [
+      "vit_base_patch16_224",
+      "vit_large_patch16_224",
+      "resnet50"
+    ],
+    "num_models": 3,
+    "score": 0.9977365975662326,
+    "submitted_time": "2026-01-15T13:27:40Z"
+  },
+  {
+    "submission_id": "0508156e232f4a2d9d4e434239971dc2",
+    "team": "blue",
+    "submitter": "anonymous",
+    "models": [
+      {
+        "model_name": "vit_base_patch16_224",
+        "source": "dummy_cache",
+        "model_parameters": null
+      },
+      {
+        "model_name": "vit_large_patch16_224",
+        "source": "dummy_cache",
+        "model_parameters": null
+      },
+      {
+        "model_name": "resnet50",
+        "source": "dummy_cache",
+        "model_parameters": null
+      }
+    ],
+    "model_names": [
+      "vit_base_patch16_224",
+      "vit_large_patch16_224",
+      "resnet50"
+    ],
+    "num_models": 3,
+    "score": 0.9977365975662326,
+    "submitted_time": "2026-01-15T13:27:49Z"
+  },
+  {
+    "submission_id": "9601269701a54891916fa923b0995ee4",
+    "team": "blue",
+    "submitter": "anonymous",
+    "models": [
+      {
+        "model_name": "vit_base_patch16_224",
+        "source": "dummy_cache",
+        "model_parameters": null
+      },
+      {
+        "model_name": "vit_large_patch16_224",
+        "source": "dummy_cache",
+        "model_parameters": null
+      },
+      {
+        "model_name": "resnet50",
+        "source": "dummy_cache",
+        "model_parameters": null
+      }
+    ],
+    "model_names": [
+      "vit_base_patch16_224",
+      "vit_large_patch16_224",
+      "resnet50"
+    ],
+    "num_models": 3,
+    "score": 0.9977365975662326,
+    "submitted_time": "2026-01-15T13:27:59Z"
+  },
+  {
+    "submission_id": "9a53e2eda7364558b07659934670b1f5",
+    "team": "blue",
+    "submitter": "anonymous",
+    "models": [
+      {
+        "model_name": "vit_base_patch16_224",
+        "source": "dummy_cache",
+        "model_parameters": null
+      },
+      {
+        "model_name": "resnet50",
+        "source": "dummy_cache",
+        "model_parameters": null
+      }
+    ],
+    "model_names": [
+      "vit_base_patch16_224",
+      "resnet50"
+    ],
+    "num_models": 2,
+    "score": 0.8369087528464697,
+    "submitted_time": "2026-01-15T13:34:24Z"
+  },
+  {
+    "submission_id": "cb8f70a0c10641e7aa1f9588400a3426",
+    "team": "blue",
+    "submitter": "anonymous",
+    "models": [
+      {
+        "model_name": "resnet50",
+        "source": "dummy_cache",
+        "model_parameters": null
+      },
+      {
+        "model_name": "resnet101",
+        "source": "dummy_cache",
+        "model_parameters": null
+      }
+    ],
+    "model_names": [
+      "resnet50",
+      "resnet101"
+    ],
+    "num_models": 2,
+    "score": 0.9984689260552404,
+    "submitted_time": "2026-01-15T13:34:35Z"
+  },
+  {
+    "submission_id": "a9dcdd432b594b138fbfbdddc71f80d3",
+    "team": "blue",
+    "submitter": "family-test-1",
+    "model_names": [
+      "resnet18",
+      "resnet34"
+    ],
+    "num_models": 2,
+    "score": 0.9255393186692762,
+    "submitted_time": "2026-01-30T18:39:20Z"
+  },
+  {
+    "submission_id": "5276d0cc1124427c9a8dca478cc7c5ef",
+    "team": "blue",
+    "submitter": "family-test-1",
+    "model_names": [
+      "resnet18",
+      "resnet34"
+    ],
+    "num_models": 2,
+    "score": 0.9255393186692762,
+    "submitted_time": "2026-01-30T20:40:27Z"
+  },
+  {
+    "submission_id": "cb824b0c5bfd41cbb928c71e1d5fab5c",
+    "team": "blue",
+    "submitter": "family-test-1",
+    "model_names": [
+      "resnet18",
+      "resnet34"
+    ],
+    "num_models": 2,
+    "score": 0.9255393186692762,
+    "submitted_time": "2026-01-30T20:41:17Z"
+  },
+  {
+    "submission_id": "b418117a57804b64a5ebf29156dc77e1",
+    "team": "blue",
+    "submitter": "family-test-1",
+    "model_names": [
+      "resnet18",
+      "resnet34"
+    ],
+    "num_models": 2,
+    "score": 0.16542392317872592,
+    "submitted_time": "2026-01-30T20:44:11Z"
+  },
+  {
+    "submission_id": "93deeb874b3e4487b9f86b7f25be1d1b",
+    "team": "blue",
+    "submitter": "family-test-1",
+    "model_names": [
+      "resnet18",
+      "resnet34"
+    ],
+    "num_models": 2,
+    "score": 0.7390657767373638,
+    "submitted_time": "2026-01-30T21:02:47Z"
+  }
+]

hackathon-data/red_submissions.json

@@ -0,0 +1,362 @@
+[
+  {
+    "submission_id": "c98476168fcc4127b74f495ef62b51fc",
+    "team": "red",
+    "submitter": "anonymous",
+    "differentiating_images": [
+      {
+        "dataset_name": "cifar100",
+        "image_identifier": "test/airplane/image_0001.png"
+      },
+      {
+        "dataset_name": "cifar100",
+        "image_identifier": "test/bear/image_0007.png"
+      },
+      {
+        "dataset_name": "cifar100",
+        "image_identifier": "test/bottle/image_0012.png"
+      }
+    ],
+    "stimuli_keys": [
+      "cifar100::test/airplane/image_0001.png",
+      "cifar100::test/bear/image_0007.png",
+      "cifar100::test/bottle/image_0012.png"
+    ],
+    "num_stimuli": 3,
+    "score": 0.0005089467322310082,
+    "submitted_time": "2026-01-15T13:28:07Z"
+  },
+  {
+    "submission_id": "00e699cc35a0427f97d0f488bd6643a6",
+    "team": "red",
+    "submitter": "anonymous",
+    "differentiating_images": [
+      {
+        "dataset_name": "cifar100",
+        "image_identifier": "test/airplane/image_0001.png"
+      },
+      {
+        "dataset_name": "cifar100",
+        "image_identifier": "test/bear/image_0007.png"
+      }
+    ],
+    "stimuli_keys": [
+      "cifar100::test/airplane/image_0001.png",
+      "cifar100::test/bear/image_0007.png"
+    ],
+    "num_stimuli": 2,
+    "score": 7.210160246629016e-10,
+    "submitted_time": "2026-01-15T13:34:47Z"
+  },
+  {
+    "submission_id": "5e08b55e1d5b4755bcdc2ac39a052586",
+    "team": "red",
+    "submitter": "anonymous",
+    "differentiating_images": [
+      {
+        "dataset_name": "cifar100",
+        "image_identifier": "test/airplane/image_0001.png"
+      },
+      {
+        "dataset_name": "cifar100",
+        "image_identifier": "test/bear/image_0007.png"
+      }
+    ],
+    "stimuli_keys": [
+      "cifar100::test/airplane/image_0001.png",
+      "cifar100::test/bear/image_0007.png"
+    ],
+    "num_stimuli": 2,
+    "score": 7.210160246629016e-10,
+    "submitted_time": "2026-01-15T13:34:58Z"
+  },
+  {
+    "submission_id": "94363c0a44584d63b9571a0e48e16a06",
+    "team": "red",
+    "submitter": "anonymous",
+    "differentiating_images": [
+      {
+        "dataset_name": "cifar100",
+        "image_identifier": "test/airplane/image_0001.png"
+      },
+      {
+        "dataset_name": "cifar100",
+        "image_identifier": "test/bear/image_0007.png"
+      },
+      {
+        "dataset_name": "cifar100",
+        "image_identifier": "test/bottle/image_0012.png"
+      }
+    ],
+    "stimuli_keys": [
+      "cifar100::test/airplane/image_0001.png",
+      "cifar100::test/bear/image_0007.png",
+      "cifar100::test/bottle/image_0012.png"
+    ],
+    "num_stimuli": 3,
+    "score": 0.032057460754414624,
+    "submitted_time": "2026-01-15T13:40:06Z"
+  },
+  {
+    "submission_id": "33cc950022a24af6bdee9c61e04a6682",
+    "team": "red",
+    "submitter": "anonymous",
+    "differentiating_images": [
+      {
+        "dataset_name": "cifar100",
+        "image_identifier": "test/airplane/image_0001.png"
+      },
+      {
+        "dataset_name": "cifar100",
+        "image_identifier": "test/bus/image_0021.png"
+      },
+      {
+        "dataset_name": "cifar100",
+        "image_identifier": "test/rocket/image_0051.png"
+      }
+    ],
+    "stimuli_keys": [
+      "cifar100::test/airplane/image_0001.png",
+      "cifar100::test/bus/image_0021.png",
+      "cifar100::test/rocket/image_0051.png"
+    ],
+    "num_stimuli": 3,
+    "score": 0.034225943844330975,
+    "submitted_time": "2026-01-15T13:40:27Z"
+  },
+  {
+    "submission_id": "a886ad9262014c5286a7a4925ceacb41",
+    "team": "red",
+    "submitter": "anonymous",
+    "differentiating_images": [
+      {
+        "dataset_name": "cifar100",
+        "image_identifier": "test/airplane/image_0001.png"
+      },
+      {
+        "dataset_name": "cifar100",
+        "image_identifier": "test/bottle/image_0012.png"
+      }
+    ],
+    "stimuli_keys": [
+      "cifar100::test/airplane/image_0001.png",
+      "cifar100::test/bottle/image_0012.png"
+    ],
+    "num_stimuli": 2,
+    "score": 1.7992535239486074e-09,
+    "submitted_time": "2026-01-15T13:49:11Z"
+  },
+  {
+    "submission_id": "0918d718f3f6460ea6e17b32daa39bb8",
+    "team": "red",
+    "submitter": "red-test-1",
+    "differentiating_images": [
+      {
+        "dataset_name": "imagenet_val",
+        "image_identifier": "ILSVRC2012_val_00000001.JPEG"
+      },
+      {
+        "dataset_name": "imagenet_val",
+        "image_identifier": "ILSVRC2012_val_00000002.JPEG"
+      }
+    ],
+    "stimuli_keys": [
+      "imagenet_val::ILSVRC2012_val_00000001.JPEG",
+      "imagenet_val::ILSVRC2012_val_00000002.JPEG"
+    ],
+    "num_stimuli": 2,
+    "score": 4.641865669618994e-09,
+    "submitted_time": "2026-02-02T17:48:57Z"
+  },
+  {
+    "submission_id": "564393f6037a487d9bac28057c4e2397",
+    "team": "red",
+    "submitter": "red-test-2",
+    "differentiating_images": [
+      {
+        "dataset_name": "imagenet_val",
+        "image_identifier": "ILSVRC2012_val_00000003.JPEG"
+      },
+      {
+        "dataset_name": "imagenet_val",
+        "image_identifier": "ILSVRC2012_val_00000004.JPEG"
+      },
+      {
+        "dataset_name": "imagenet_val",
+        "image_identifier": "ILSVRC2012_val_00000005.JPEG"
+      }
+    ],
+    "stimuli_keys": [
+      "imagenet_val::ILSVRC2012_val_00000003.JPEG",
+      "imagenet_val::ILSVRC2012_val_00000004.JPEG",
+      "imagenet_val::ILSVRC2012_val_00000005.JPEG"
+    ],
+    "num_stimuli": 3,
+    "score": 0.0539067506733818,
+    "submitted_time": "2026-02-02T17:48:58Z"
+  },
+  {
+    "submission_id": "f3d164ddd6584bf284a403dd06e2c0d8",
+    "team": "red",
+    "submitter": "red-test-3",
+    "differentiating_images": [
+      {
+        "dataset_name": "imagenet_val",
+        "image_identifier": "ILSVRC2012_val_00000002.JPEG"
+      },
+      {
+        "dataset_name": "imagenet_val",
+        "image_identifier": "ILSVRC2012_val_00000006.JPEG"
+      }
+    ],
+    "stimuli_keys": [
+      "imagenet_val::ILSVRC2012_val_00000002.JPEG",
+      "imagenet_val::ILSVRC2012_val_00000006.JPEG"
+    ],
+    "num_stimuli": 2,
+    "score": 3.537208304038586e-09,
+    "submitted_time": "2026-02-02T17:48:59Z"
+  },
+  {
+    "submission_id": "89ef5cae40e2470eaec98fbfbb568545",
+    "team": "red",
+    "submitter": "red-test-1",
+    "differentiating_images": [
+      {
+        "dataset_name": "imagenet_val",
+        "image_identifier": "ILSVRC2012_val_00000001.JPEG"
+      },
+      {
+        "dataset_name": "imagenet_val",
+        "image_identifier": "ILSVRC2012_val_00000002.JPEG"
+      }
+    ],
+    "stimuli_keys": [
+      "imagenet_val::ILSVRC2012_val_00000001.JPEG",
+      "imagenet_val::ILSVRC2012_val_00000002.JPEG"
+    ],
+    "num_stimuli": 2,
+    "score": 4.641865669618994e-09,
+    "submitted_time": "2026-02-02T18:22:19Z"
+  },
+  {
+    "submission_id": "a95f3e9e9c8844b587297c96c4225292",
+    "team": "red",
+    "submitter": "red-test-2",
+    "differentiating_images": [
+      {
+        "dataset_name": "imagenet_val",
+        "image_identifier": "ILSVRC2012_val_00000003.JPEG"
+      },
+      {
+        "dataset_name": "imagenet_val",
+        "image_identifier": "ILSVRC2012_val_00000004.JPEG"
+      },
+      {
+        "dataset_name": "imagenet_val",
+        "image_identifier": "ILSVRC2012_val_00000005.JPEG"
+      }
+    ],
+    "stimuli_keys": [
+      "imagenet_val::ILSVRC2012_val_00000003.JPEG",
+      "imagenet_val::ILSVRC2012_val_00000004.JPEG",
+      "imagenet_val::ILSVRC2012_val_00000005.JPEG"
+    ],
+    "num_stimuli": 3,
+    "score": 0.0539067506733818,
+    "submitted_time": "2026-02-02T18:22:19Z"
+  },
+  {
+    "submission_id": "ce09c58fe963407e80adeee3c598e049",
+    "team": "red",
+    "submitter": "red-test-3",
+    "differentiating_images": [
+      {
+        "dataset_name": "imagenet_val",
+        "image_identifier": "ILSVRC2012_val_00000002.JPEG"
+      },
+      {
+        "dataset_name": "imagenet_val",
+        "image_identifier": "ILSVRC2012_val_00000006.JPEG"
+      }
+    ],
+    "stimuli_keys": [
+      "imagenet_val::ILSVRC2012_val_00000002.JPEG",
+      "imagenet_val::ILSVRC2012_val_00000006.JPEG"
+    ],
+    "num_stimuli": 2,
+    "score": 3.537208304038586e-09,
+    "submitted_time": "2026-02-02T18:22:20Z"
+  },
+  {
+    "submission_id": "e14d25b38a2a43299ccc5a4aaa7a8095",
+    "team": "red",
+    "submitter": "red-test-1",
+    "differentiating_images": [
+      {
+        "dataset_name": "imagenet_val",
+        "image_identifier": "ILSVRC2012_val_00000001.JPEG"
+      },
+      {
+        "dataset_name": "imagenet_val",
+        "image_identifier": "ILSVRC2012_val_00000002.JPEG"
+      }
+    ],
+    "stimuli_keys": [
+      "imagenet_val::ILSVRC2012_val_00000001.JPEG",
+      "imagenet_val::ILSVRC2012_val_00000002.JPEG"
+    ],
+    "num_stimuli": 2,
+    "score": 4.641865669618994e-09,
+    "submitted_time": "2026-02-03T16:59:12Z"
+  },
+  {
+    "submission_id": "9f5b6f0a8e484796b5fb49f6a8e73978",
+    "team": "red",
+    "submitter": "red-test-2",
+    "differentiating_images": [
+      {
+        "dataset_name": "imagenet_val",
+        "image_identifier": "ILSVRC2012_val_00000003.JPEG"
+      },
+      {
+        "dataset_name": "imagenet_val",
+        "image_identifier": "ILSVRC2012_val_00000004.JPEG"
+      },
+      {
+        "dataset_name": "imagenet_val",
+        "image_identifier": "ILSVRC2012_val_00000005.JPEG"
+      }
+    ],
+    "stimuli_keys": [
+      "imagenet_val::ILSVRC2012_val_00000003.JPEG",
+      "imagenet_val::ILSVRC2012_val_00000004.JPEG",
+      "imagenet_val::ILSVRC2012_val_00000005.JPEG"
+    ],
+    "num_stimuli": 3,
+    "score": 0.0539067506733818,
+    "submitted_time": "2026-02-03T16:59:15Z"
+  },
+  {
+    "submission_id": "3cfc065bb85a40e28927d950b8cb156a",
+    "team": "red",
+    "submitter": "red-test-3",
+    "differentiating_images": [
+      {
+        "dataset_name": "imagenet_val",
+        "image_identifier": "ILSVRC2012_val_00000002.JPEG"
+      },
+      {
+        "dataset_name": "imagenet_val",
+        "image_identifier": "ILSVRC2012_val_00000006.JPEG"
+      }
+    ],
+    "stimuli_keys": [
+      "imagenet_val::ILSVRC2012_val_00000002.JPEG",
+      "imagenet_val::ILSVRC2012_val_00000006.JPEG"
+    ],
+    "num_stimuli": 2,
+    "score": 3.537208304038586e-09,
+    "submitted_time": "2026-02-03T16:59:15Z"
+  }
+]

pyproject.toml

@@ -0,0 +1,13 @@
+[tool.ruff]
+# Enable pycodestyle (`E`) and Pyflakes (`F`) codes by default.
+select = ["E", "F"]
+ignore = ["E501"] # line too long (black is taking care of this)
+line-length = 119
+fixable = ["A", "B", "C", "D", "E", "F", "G", "I", "N", "Q", "S", "T", "W", "ANN", "ARG", "BLE", "COM", "DJ", "DTZ", "EM", "ERA", "EXE", "FBT", "ICN", "INP", "ISC", "NPY", "PD", "PGH", "PIE", "PL", "PT", "PTH", "PYI", "RET", "RSE", "RUF", "SIM", "SLF", "TCH", "TID", "TRY", "UP", "YTT"]
+
+[tool.isort]
+profile = "black"
+line_length = 119
+
+[tool.black]
+line-length = 119

requirements.txt

@@ -0,0 +1,18 @@
+APScheduler
+black
+datasets
+modal
+gradio
+gradio[oauth]
+gradio_leaderboard==0.0.13
+gradio_client
+huggingface-hub>=0.18.0
+matplotlib
+python-dotenv
+numpy
+pandas
+python-dateutil
+tqdm
+transformers
+tokenizers>=0.15.0
+sentencepiece

scripts/blue_family_smoke_test.py

@@ -0,0 +1,100 @@
+from __future__ import annotations
+
+import argparse
+import json
+import os
+from pathlib import Path
+
+import modal
+
+from dotenv import load_dotenv
+load_dotenv()
+
+
+def _require_env(name: str) -> str:
+    value = os.environ.get(name, "").strip()
+    if not value:
+        raise ValueError(f"Missing required env var: {name}")
+    return value
+
+
+def _seed_dummy_dataset(app_name: str) -> None:
+    seed_fn = modal.Function.from_name(app_name, "seed_dummy_dataset")
+    seed = seed_fn.remote(num_images=6, image_size=224, dataset_name="dummy")
+    print(f"Seeded dataset at {seed['dataset_root']}")
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Blue family smoke test")
+    parser.add_argument(
+        "--stimuli",
+        type=str,
+        default=None,
+        help="Path to custom stimuli JSONL file. If not provided, uses HACKATHON_STIMULI_CATALOG env var.",
+    )
+    parser.add_argument(
+        "--s3",
+        action="store_true",
+        help="Use S3-backed datasets (requires aws-s3-credentials Modal secret).",
+    )
+    parser.add_argument(
+        "--skip-seed",
+        action="store_true",
+        help="Skip seeding dummy dataset (use when testing with real images).",
+    )
+    args = parser.parse_args()
+
+    _require_env("HACKATHON_MODAL_ENABLE")
+    _require_env("HACKATHON_MODEL_REGISTRY")
+    data_dir = _require_env("HACKATHON_DATA_DIR")
+
+    # Use custom stimuli if provided, otherwise use env var
+    if args.stimuli:
+        stimuli_path = Path(args.stimuli).resolve()
+        if not stimuli_path.exists():
+            raise ValueError(f"Stimuli file not found: {stimuli_path}")
+        os.environ["HACKATHON_STIMULI_CATALOG"] = str(stimuli_path)
+        print(f"Using custom stimuli: {stimuli_path}")
+    else:
+        _require_env("HACKATHON_STIMULI_CATALOG")
+
+    # Set S3 mode if requested
+    if args.s3:
+        os.environ["HACKATHON_USE_S3"] = "true"
+        print("S3 mode enabled - will use extract_embeddings_s3 function")
+
+    app_name = os.environ.get("HACKATHON_MODAL_APP", "iclr2026-eval")
+
+    # Only seed dummy dataset if not using real images
+    if not args.skip_seed and not args.s3:
+        _seed_dummy_dataset(app_name)
+    elif args.skip_seed:
+        print("Skipping dummy dataset seeding")
+    elif args.s3:
+        print("S3 mode: skipping dummy dataset seeding (using real images)")
+
+    from app import submit_blue
+
+    model_sets = [
+        # ["resnet18", "mobilenet_v2", "densenet121"],
+        # ["vgg16", "resnet18"],
+        # ["mobilenet_v2", "densenet121", "vgg16"],
+        ["resnet18", "resnet34"],
+    ]
+
+    for idx, models in enumerate(model_sets, start=1):
+        payload = json.dumps({"models": models})
+        submitter = f"family-test-{idx}"
+        msg, leaderboard, pairwise = submit_blue(submitter, payload)
+        print(f"Submission {idx} message: {msg}")
+        print(f"Submission {idx} leaderboard: {leaderboard.tail(1).to_dict(orient='records')}")
+        print(f"Submission {idx} pairwise: {pairwise.to_dict(orient='records')}")
+        assert not pairwise.empty, "Pairwise table should not be empty."
+
+    # blue_path = Path(data_dir) / "blue_submissions.json"
+    # assert blue_path.exists(), f"Missing submission file: {blue_path}"
+    print("Blue family smoke test complete.")
+
+
+if __name__ == "__main__":
+    main()

scripts/blue_team_submit.py

@@ -0,0 +1,107 @@
+from __future__ import annotations
+
+import argparse
+import json
+import os
+from pathlib import Path
+
+import modal
+
+from dotenv import load_dotenv
+load_dotenv()
+
+def _require_env(name: str) -> str:
+    value = os.environ.get(name, "").strip()
+    if not value:
+        raise ValueError(f"Missing required env var: {name}")
+    return value
+
+
+def _seed_dummy_dataset(app_name: str) -> None:
+    seed_fn = modal.Function.from_name(app_name, "seed_dummy_dataset")
+    seed = seed_fn.remote(num_images=6, image_size=224, dataset_name="dummy")
+    print(f"Seeded dataset at {seed['dataset_root']}")
+
+
+def main(
+    models: list[str], 
+    submitter: str,
+    stimuli: str | None, 
+    use_s3: bool = True, 
+    skip_seed: bool = False
+) -> None:
+
+    _require_env("HACKATHON_MODAL_ENABLE")
+    _require_env("HACKATHON_MODEL_REGISTRY")
+
+    # Use custom stimuli if provided, otherwise use env var
+    if stimuli:
+        stimuli_path = Path(stimuli).resolve()
+        if not stimuli_path.exists():
+            raise ValueError(f"Stimuli file not found: {stimuli_path}")
+        os.environ["HACKATHON_STIMULI_CATALOG"] = str(stimuli_path)
+        print(f"Using custom stimuli: {stimuli_path}")
+    else:
+        _require_env("HACKATHON_STIMULI_CATALOG")
+
+    # Set S3 mode if requested
+    if use_s3:
+        os.environ["HACKATHON_USE_S3"] = "true"
+        print("S3 mode enabled - will use extract_embeddings_s3 function")
+
+    app_name = os.environ.get("HACKATHON_MODAL_APP", "iclr2026-eval")
+
+    # Only seed dummy dataset if not using real images
+    if not skip_seed and not use_s3:
+        _seed_dummy_dataset(app_name)
+    elif skip_seed:
+        print("Skipping dummy dataset seeding")
+    elif use_s3:
+        print("S3 mode: skipping dummy dataset seeding (using real images)")
+
+    from app import submit_blue
+
+    payload = json.dumps({"models": models})
+    msg, leaderboard, pairwise = submit_blue(submitter, payload)
+    print(f"Submission message: {msg}")
+    print(f"Submission leaderboard: {leaderboard.tail(1).to_dict(orient='records')}")
+    print(f"Submission pairwise: {pairwise.to_dict(orient='records')}")
+    assert not pairwise.empty, "Pairwise table should not be empty."
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Blue family smoke test")
+    parser.add_argument(
+        "--models",
+        type=str,
+        nargs="+",
+        default=None,
+        help="List of models to submit. If not provided, uses predefined model sets.",
+    )
+    parser.add_argument(
+        "--submitter",
+        type=str,
+        default="test",
+        help="Submitter name to use for the submission.",
+    )
+    parser.add_argument(
+        "--stimuli",
+        type=str,
+        default=None,
+        help="Path to custom stimuli JSONL file. If not provided, uses HACKATHON_STIMULI_CATALOG env var.",
+    )
+    parser.add_argument(
+        "--s3",
+        action="store_true",
+        help="Use S3-backed datasets (requires aws-s3-credentials Modal secret).",
+    )
+    parser.add_argument(
+        "--skip-seed",
+        action="store_true",
+        help="Skip seeding dummy dataset (use when testing with real images).",
+    )
+    args = parser.parse_args()
+
+    models = args.models if args.models is not None else ["resnet18", "resnet34"]
+    submitter = args.submitter
+
+    main(models, submitter, args.stimuli, args.s3, args.skip_seed)

scripts/generate_blue_registry.py

@@ -0,0 +1,225 @@
+#!/usr/bin/env python3
+"""Generate blue_team_model_registry.json from one_model_per_family.json.
+
+For each model, queries timm to auto-detect:
+  - preprocessing hyperparameters (resize, crop, mean, std)
+  - the appropriate embedding layer name
+  - embedding strategy
+
+Usage (local):
+    python scripts/generate_blue_registry.py
+
+Usage (Modal):
+    modal run scripts/generate_blue_registry.py
+"""
+from __future__ import annotations
+
+import json
+import math
+import sys
+from pathlib import Path
+from typing import Any
+
+CONFIGS_DIR = Path(__file__).resolve().parent.parent / "configs"
+INPUT_PATH = CONFIGS_DIR / "one_model_per_family.json"
+OUTPUT_PATH = CONFIGS_DIR / "blue_team_model_registry.json"
+
+
+def detect_embedding_layer(model_name: str, model: Any) -> str:
+    """Detect the best layer for embedding extraction by inspecting model modules."""
+    import torch.nn as nn
+
+    modules = dict(model.named_modules())
+
+    # Priority 1: head.global_pool (hybrid/newer architectures)
+    if "head.global_pool" in modules:
+        m = modules["head.global_pool"]
+        if not isinstance(m, nn.Identity):
+            return "head.global_pool"
+
+    # Priority 2: global_pool (standard CNNs)
+    if "global_pool" in modules:
+        m = modules["global_pool"]
+        if not isinstance(m, nn.Identity):
+            return "global_pool"
+
+    # Priority 3: pooling (ConvMixer uses SelectAdaptivePool2d as 'pooling')
+    if "pooling" in modules:
+        m = modules["pooling"]
+        if not isinstance(m, nn.Identity):
+            return "pooling"
+
+    # Priority 4: fc_norm (some ViT variants like EVA, BEiT, AIMv2)
+    if "fc_norm" in modules:
+        return "fc_norm"
+
+    # Priority 5: norm (ViT/transformer final normalization before head)
+    # Accept any normalization-like module, but NOT ModuleList (e.g. CrossViT)
+    if "norm" in modules:
+        m = modules["norm"]
+        if not isinstance(m, (nn.ModuleList, nn.ModuleDict)):
+            return "norm"
+
+    # Priority 6: norm.1 (CrossViT has ModuleList of norms; use the large-scale branch)
+    if "norm.1" in modules:
+        return "norm.1"
+
+    # Priority 7: norm4 (CoaT models have norm1..norm4 for each stage)
+    if "norm4" in modules:
+        return "norm4"
+
+    # Priority 8: head.norm
+    if "head.norm" in modules:
+        return "head.norm"
+
+    # Priority 9: stages.3.norm (PVT v2 and similar hierarchical transformers)
+    if "stages.3.norm" in modules:
+        return "stages.3.norm"
+
+    # Priority 10: head.head.bn (RepViT: BatchNorm1d before classifier Linear)
+    if "head.head.bn" in modules:
+        return "head.head.bn"
+
+    # Priority 11: head.bn (LeViT/NormLinear: BatchNorm1d before classifier Linear)
+    if "head.bn" in modules:
+        return "head.bn"
+
+    # Priority 12: head (last resort - may output logits)
+    if "head" in modules:
+        return "head"
+
+    # Priority 12: avgpool (some torchvision-style models)
+    if "avgpool" in modules:
+        return "avgpool"
+
+    raise ValueError(f"Could not detect embedding layer for {model_name}")
+
+
+def generate_spec(model_name: str) -> dict[str, Any]:
+    """Generate a full model spec dict for a given timm model name."""
+    import timm
+    import timm.data
+
+    model = timm.create_model(model_name, pretrained=False)
+    data_config = timm.data.resolve_data_config(model=model)
+
+    input_size = data_config["input_size"]  # (C, H, W)
+    crop = input_size[1]  # Use height (should == width for these models)
+    crop_pct = data_config.get("crop_pct", 0.875)
+    resize = round(crop / crop_pct)
+    mean = [round(v, 3) for v in data_config["mean"]]
+    std = [round(v, 3) for v in data_config["std"]]
+
+    layer = detect_embedding_layer(model_name, model)
+
+    return {
+        "model_name": model_name,
+        "source": "timm",
+        "weights": "imagenet",
+        "layer": layer,
+        "embedding": "flatten",
+        "preprocess": {
+            "resize": resize,
+            "crop": crop,
+            "mean": mean,
+            "std": std,
+        },
+    }
+
+
+def validate_spec(spec: dict[str, Any]) -> tuple[bool, str]:
+    """Validate a spec by doing a forward pass with a dummy image."""
+    import torch
+    import timm
+
+    model_name = spec["model_name"]
+    layer = spec["layer"]
+    crop = spec["preprocess"]["crop"]
+
+    try:
+        model = timm.create_model(model_name, pretrained=False)
+        model.eval()
+
+        modules = dict(model.named_modules())
+        if layer not in modules:
+            return False, f"Layer '{layer}' not found in model"
+
+        target_module = modules[layer]
+        outputs = []
+
+        def hook(_module, _inputs, output):
+            outputs.append(output)
+
+        handle = target_module.register_forward_hook(hook)
+
+        dummy_input = torch.randn(1, 3, crop, crop)
+        with torch.no_grad():
+            model(dummy_input)
+
+        handle.remove()
+
+        if not outputs:
+            return False, "No output captured from hook"
+
+        out = outputs[0]
+        if isinstance(out, (tuple, list)):
+            out = out[0]
+        if isinstance(out, dict):
+            out = out.get("last_hidden_state") or out.get("pooler_output")
+        if not isinstance(out, torch.Tensor):
+            return False, f"Hook output is not a tensor: {type(out)}"
+
+        # Apply flatten strategy
+        if out.ndim == 2:
+            dim = out.shape[1]
+        elif out.ndim >= 3:
+            dim = 1
+            for d in out.shape[1:]:
+                dim *= d
+        else:
+            dim = out.numel()
+
+        return True, f"OK (dim={dim})"
+
+    except Exception as e:
+        return False, f"Error: {e}"
+
+
+def main() -> None:
+    model_names = json.loads(INPUT_PATH.read_text())
+    print(f"Generating specs for {len(model_names)} models from {INPUT_PATH.name}")
+
+    specs = []
+    errors = []
+
+    for i, name in enumerate(model_names):
+        print(f"[{i+1}/{len(model_names)}] {name}...", end=" ", flush=True)
+        try:
+            spec = generate_spec(name)
+            # Validate by doing a forward pass
+            ok, msg = validate_spec(spec)
+            if ok:
+                specs.append(spec)
+                print(f"layer={spec['layer']} resize={spec['preprocess']['resize']} "
+                      f"crop={spec['preprocess']['crop']} {msg}")
+            else:
+                errors.append((name, msg))
+                print(f"VALIDATION FAILED: {msg}")
+        except Exception as e:
+            errors.append((name, str(e)))
+            print(f"FAILED: {e}")
+
+    OUTPUT_PATH.write_text(json.dumps(specs, indent=2) + "\n")
+    print(f"\nWrote {len(specs)} specs to {OUTPUT_PATH}")
+
+    if errors:
+        print(f"\n{len(errors)} errors:")
+        for name, msg in errors:
+            print(f"  {name}: {msg}")
+        sys.exit(1)
+    else:
+        print(f"\nAll {len(specs)} models generated and validated successfully.")
+
+
+if __name__ == "__main__":
+    main()

scripts/pipeline_smoke_test.py

@@ -0,0 +1,70 @@
+from __future__ import annotations
+
+import json
+import os
+from pathlib import Path
+
+import modal
+
+
+def _require_env(name: str) -> str:
+    value = os.environ.get(name, "").strip()
+    if not value:
+        raise ValueError(f"Missing required env var: {name}")
+    return value
+
+
+def main() -> None:
+    _require_env("HACKATHON_MODAL_ENABLE")
+    _require_env("HACKATHON_MODEL_REGISTRY")
+    _require_env("HACKATHON_STIMULI_CATALOG")
+    data_dir = _require_env("HACKATHON_DATA_DIR")
+
+    app_name = os.environ.get("HACKATHON_MODAL_APP", "iclr2026-eval")
+    seed_fn = modal.Function.from_name(app_name, "seed_dummy_dataset")
+    seed = seed_fn.remote(num_images=6, image_size=224, dataset_name="dummy")
+    print(f"Seeded dataset at {seed['dataset_root']}")
+
+    from app import submit_blue, submit_red
+
+    blue_payload = json.dumps({"models": ["resnet18", "resnet34"]})
+    red_payload = json.dumps(
+        {
+            "differentiating_images": [
+                {"dataset_name": "dummy", "image_identifier": "images/img_0000.png"},
+                {"dataset_name": "dummy", "image_identifier": "images/img_0001.png"},
+            ]
+        }
+    )
+
+    blue_msg, blue_leaderboard, blue_pairwise = submit_blue("pipeline-test", blue_payload)
+    print("Blue message:", blue_msg)
+    print("Blue leaderboard:", blue_leaderboard.tail(1).to_dict(orient="records"))
+    print("Blue pairwise:", blue_pairwise.to_dict(orient="records"))
+
+    assert not blue_leaderboard.empty, "Blue leaderboard should not be empty."
+    assert not blue_pairwise.empty, "Blue pairwise table should not be empty."
+
+    red_msg, red_leaderboard, red_pairwise = submit_red("pipeline-test", red_payload)
+    print("Red message:", red_msg)
+    print("Red leaderboard:", red_leaderboard.tail(1).to_dict(orient="records"))
+    print("Red pairwise:", red_pairwise.to_dict(orient="records"))
+
+    assert not red_leaderboard.empty, "Red leaderboard should not be empty."
+    assert not red_pairwise.empty, "Red pairwise table should not be empty."
+
+    for df in (blue_pairwise, red_pairwise):
+        cka_vals = df["CKA"].astype(float)
+        assert (cka_vals >= -1e-3).all(), "CKA should be >= 0."
+        assert (cka_vals <= 1.0001).all(), "CKA should be <= 1."
+
+    blue_path = Path(data_dir) / "blue_submissions.json"
+    red_path = Path(data_dir) / "red_submissions.json"
+    assert blue_path.exists(), f"Missing submission file: {blue_path}"
+    assert red_path.exists(), f"Missing submission file: {red_path}"
+
+    print("Submission pipeline smoke test complete.")
+
+
+if __name__ == "__main__":
+    main()

scripts/red_team_smoke_test.py

@@ -0,0 +1,143 @@
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import sys
+from pathlib import Path
+
+# Add project root to path so we can import app module
+PROJECT_ROOT = Path(__file__).resolve().parent.parent
+sys.path.insert(0, str(PROJECT_ROOT))
+
+import modal
+
+from dotenv import load_dotenv
+load_dotenv()
+
+
+def _require_env(name: str) -> str:
+    value = os.environ.get(name, "").strip()
+    if not value:
+        raise ValueError(f"Missing required env var: {name}")
+    return value
+
+
+def _seed_dummy_dataset(app_name: str) -> None:
+    seed_fn = modal.Function.from_name(app_name, "seed_dummy_dataset")
+    seed = seed_fn.remote(num_images=6, image_size=224, dataset_name="dummy")
+    print(f"Seeded dataset at {seed['dataset_root']}")
+
+
+def _load_stimuli_from_catalog(stimuli_path: str) -> list[dict[str, str]]:
+    """Load stimuli from a JSONL catalog file."""
+    stimuli = []
+    with open(stimuli_path, "r") as f:
+        for line in f:
+            line = line.strip()
+            if line:
+                stimuli.append(json.loads(line))
+    return stimuli
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Red team smoke test")
+    parser.add_argument(
+        "--stimuli",
+        type=str,
+        default=None,
+        help="Path to custom stimuli JSONL file. If not provided, uses HACKATHON_STIMULI_CATALOG env var.",
+    )
+    parser.add_argument(
+        "--s3",
+        action="store_true",
+        help="Use S3-backed datasets (requires aws-s3-credentials Modal secret).",
+    )
+    parser.add_argument(
+        "--skip-seed",
+        action="store_true",
+        help="Skip seeding dummy dataset (use when testing with real images).",
+    )
+    args = parser.parse_args()
+
+    _require_env("HACKATHON_MODAL_ENABLE")
+    _require_env("HACKATHON_MODEL_REGISTRY")
+    data_dir = _require_env("HACKATHON_DATA_DIR")
+
+    # Use custom stimuli if provided, otherwise use env var
+    if args.stimuli:
+        stimuli_path = Path(args.stimuli).resolve()
+        if not stimuli_path.exists():
+            raise ValueError(f"Stimuli file not found: {stimuli_path}")
+        os.environ["HACKATHON_STIMULI_CATALOG"] = str(stimuli_path)
+        print(f"Using custom stimuli: {stimuli_path}")
+    else:
+        _require_env("HACKATHON_STIMULI_CATALOG")
+
+    stimuli_catalog_path = os.environ.get("HACKATHON_STIMULI_CATALOG", "").strip()
+
+    # Set S3 mode if requested
+    if args.s3:
+        os.environ["HACKATHON_USE_S3"] = "true"
+        print("S3 mode enabled - will use extract_embeddings_s3 function")
+
+    app_name = os.environ.get("HACKATHON_MODAL_APP", "iclr2026-eval")
+
+    # Only seed dummy dataset if not using real images
+    if not args.skip_seed and not args.s3:
+        _seed_dummy_dataset(app_name)
+    elif args.skip_seed:
+        print("Skipping dummy dataset seeding")
+    elif args.s3:
+        print("S3 mode: skipping dummy dataset seeding (using real images)")
+
+    from app import submit_red
+
+    # Build stimulus sets based on mode
+    if args.s3:
+        # Load stimuli from catalog for S3 mode
+        catalog_stimuli = _load_stimuli_from_catalog(stimuli_catalog_path)
+        if len(catalog_stimuli) < 6:
+            raise ValueError(f"Need at least 6 stimuli in catalog, found {len(catalog_stimuli)}")
+        
+        # Create test sets from real catalog stimuli
+        stimulus_sets = [
+            catalog_stimuli[0:2],  # First 2 stimuli
+            catalog_stimuli[2:5],  # Next 3 stimuli
+            [catalog_stimuli[1], catalog_stimuli[5]],  # Mixed selection
+        ]
+        print(f"Using {len(catalog_stimuli)} stimuli from catalog for S3 mode")
+    else:
+        # Use dummy stimuli for local mode
+        stimulus_sets = [
+            [
+                {"dataset_name": "dummy", "image_identifier": "images/img_0000.png"},
+                {"dataset_name": "dummy", "image_identifier": "images/img_0001.png"},
+            ],
+            [
+                {"dataset_name": "dummy", "image_identifier": "images/img_0002.png"},
+                {"dataset_name": "dummy", "image_identifier": "images/img_0003.png"},
+                {"dataset_name": "dummy", "image_identifier": "images/img_0004.png"},
+            ],
+            [
+                {"dataset_name": "dummy", "image_identifier": "images/img_0001.png"},
+                {"dataset_name": "dummy", "image_identifier": "images/img_0005.png"},
+            ],
+        ]
+
+    for idx, stimuli in enumerate(stimulus_sets, start=1):
+        payload = json.dumps({"differentiating_images": stimuli})
+        submitter = f"red-test-{idx}"
+        msg, leaderboard, pairwise = submit_red(submitter, payload)
+        print(f"Submission {idx} message: {msg}")
+        print(f"Submission {idx} leaderboard: {leaderboard.tail(1).to_dict(orient='records')}")
+        print(f"Submission {idx} pairwise: {pairwise.to_dict(orient='records')}")
+        assert not pairwise.empty, "Pairwise table should not be empty."
+
+    red_path = Path(data_dir) / "red_submissions.json"
+    assert red_path.exists(), f"Missing submission file: {red_path}"
+    print("Red team smoke test complete.")
+
+
+if __name__ == "__main__":
+    main()

scripts/run_local.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+
+# Prefer cluster scratch if available; fall back to a local cache in the repo.
+if [ -n "${CACHE_ROOT:-}" ]; then
+  CACHE_BASE="$CACHE_ROOT"
+elif [ -d "/orcd/scratch/bcs/001" ]; then
+  CACHE_BASE="/orcd/scratch/bcs/001/$USER"
+else
+  CACHE_BASE="${PROJECT_ROOT}/.cache"
+fi
+
+export HF_HOME="${HF_HOME:-${CACHE_BASE}/.cache/huggingface}"
+export TORCH_HOME="${TORCH_HOME:-${CACHE_BASE}/.cache/torch}"
+
+mkdir -p "$HF_HOME" "$TORCH_HOME"
+
+exec python "${PROJECT_ROOT}/app.py"

scripts/smoke_test_registry.py

@@ -0,0 +1,230 @@
+#!/usr/bin/env python3
+"""Smoke test to validate model registry specs on Modal.
+
+For each model in the registry:
+  1. Creates the model via timm (pretrained=False for speed)
+  2. Builds the preprocessing transform from the spec
+  3. Runs a forward pass with a dummy image
+  4. Registers a hook on the specified layer
+  5. Verifies the embedding output is a valid tensor with expected dimensions
+
+Usage (local, no GPU needed for smoke test):
+    python scripts/smoke_test_registry.py --registry configs/blue_team_model_registry.json
+
+Usage (Modal):
+    modal run scripts/smoke_test_registry.py --registry configs/blue_team_model_registry.json
+
+Usage (validate red team registry — from private eval-backend repo):
+    python scripts/smoke_test_registry.py --registry /path/to/eval-backend/configs/red_team_model_registry.json
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+import time
+from pathlib import Path
+from typing import Any
+
+
+def validate_single_model(spec: dict[str, Any]) -> tuple[bool, str, dict[str, Any]]:
+    """Validate a single model spec by loading it and running a forward pass.
+
+    Returns (success, message, details).
+    """
+    import torch
+    import timm
+    from torchvision import transforms
+
+    model_name = spec["model_name"]
+    layer = spec["layer"]
+    embedding = spec.get("embedding", "flatten")
+    preprocess = spec.get("preprocess", {})
+
+    details: dict[str, Any] = {
+        "model_name": model_name,
+        "layer": layer,
+        "embedding": embedding,
+    }
+
+    try:
+        # 1. Create model
+        t0 = time.time()
+        model = timm.create_model(model_name, pretrained=False)
+        model.eval()
+        details["model_load_time"] = round(time.time() - t0, 2)
+
+        # 2. Verify layer exists
+        modules = dict(model.named_modules())
+        if layer not in modules:
+            available = [n for n in modules.keys() if n][:20]
+            return False, f"Layer '{layer}' not found. Available (first 20): {available}", details
+
+        target_module = modules[layer]
+
+        # 3. Build transform from spec
+        resize = preprocess.get("resize")
+        crop = preprocess.get("crop")
+        mean = preprocess.get("mean", [0.485, 0.456, 0.406])
+        std = preprocess.get("std", [0.229, 0.224, 0.225])
+
+        ops = []
+        if resize:
+            ops.append(transforms.Resize((resize, resize)))
+        if crop:
+            ops.append(transforms.CenterCrop(crop))
+        ops.extend([transforms.ToTensor(), transforms.Normalize(mean=mean, std=std)])
+        transform = transforms.Compose(ops)
+
+        # 4. Create dummy input and run forward pass with hook
+        from PIL import Image
+        import numpy as np
+
+        # Create a dummy RGB image larger than any resize
+        dummy_size = max(resize or 224, crop or 224, 224) + 32
+        dummy_img = Image.fromarray(
+            np.random.randint(0, 256, (dummy_size, dummy_size, 3), dtype=np.uint8)
+        )
+        input_tensor = transform(dummy_img).unsqueeze(0)
+        details["input_shape"] = list(input_tensor.shape)
+
+        outputs = []
+
+        def hook(_module, _inputs, output):
+            outputs.append(output)
+
+        handle = target_module.register_forward_hook(hook)
+
+        t0 = time.time()
+        with torch.no_grad():
+            model(input_tensor)
+        details["forward_time"] = round(time.time() - t0, 2)
+
+        handle.remove()
+
+        # 5. Validate hook output
+        if not outputs:
+            return False, "No output captured from hook", details
+
+        out = outputs[0]
+        if isinstance(out, (tuple, list)):
+            out = out[0]
+        if isinstance(out, dict):
+            out = out.get("last_hidden_state") or out.get("pooler_output")
+        if not isinstance(out, torch.Tensor):
+            return False, f"Hook output is not a tensor: {type(out).__name__}", details
+
+        details["raw_output_shape"] = list(out.shape)
+
+        # 6. Apply embedding strategy
+        if out.ndim == 2:
+            embedded = out
+        elif out.ndim >= 3:
+            if embedding == "cls":
+                embedded = out[:, 0]
+            elif embedding == "pool":
+                embedded = out.mean(dim=tuple(range(2, out.ndim)))
+            elif embedding == "mean":
+                embedded = out.mean(dim=tuple(range(1, out.ndim)))
+            else:  # flatten
+                embedded = out.flatten(start_dim=1)
+        else:
+            embedded = out.flatten(start_dim=1)
+
+        details["embedding_shape"] = list(embedded.shape)
+        details["embedding_dim"] = int(embedded.shape[1]) if embedded.ndim == 2 else int(embedded.numel())
+
+        # 7. Sanity checks
+        if embedded.ndim != 2:
+            return False, f"Embedding is not 2D after strategy: shape={embedded.shape}", details
+        if embedded.shape[0] != 1:
+            return False, f"Batch dimension mismatch: {embedded.shape[0]}", details
+        if embedded.shape[1] == 0:
+            return False, "Embedding dimension is 0", details
+        if torch.isnan(embedded).any():
+            return False, "Embedding contains NaN values", details
+        if torch.isinf(embedded).any():
+            return False, "Embedding contains Inf values", details
+
+        return True, f"OK (dim={details['embedding_dim']})", details
+
+    except Exception as e:
+        return False, f"Error: {e}", details
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Smoke test model registry specs")
+    parser.add_argument(
+        "--registry",
+        type=str,
+        required=True,
+        help="Path to model registry JSON file",
+    )
+    parser.add_argument(
+        "--model",
+        type=str,
+        default=None,
+        help="Test only a specific model by name (for debugging)",
+    )
+    parser.add_argument(
+        "--output",
+        type=str,
+        default=None,
+        help="Path to write detailed results JSON",
+    )
+    args = parser.parse_args()
+
+    registry_path = Path(args.registry)
+    specs = json.loads(registry_path.read_text())
+    if isinstance(specs, dict):
+        specs = specs.get("models", specs)
+
+    if args.model:
+        specs = [s for s in specs if s["model_name"] == args.model]
+        if not specs:
+            print(f"Model '{args.model}' not found in registry")
+            sys.exit(1)
+
+    print(f"Smoke testing {len(specs)} models from {registry_path.name}")
+    print("=" * 70)
+
+    results = []
+    passed = 0
+    failed = 0
+
+    for i, spec in enumerate(specs):
+        name = spec["model_name"]
+        print(f"[{i+1}/{len(specs)}] {name}...", end=" ", flush=True)
+
+        ok, msg, details = validate_single_model(spec)
+        details["passed"] = ok
+        details["message"] = msg
+        results.append(details)
+
+        if ok:
+            passed += 1
+            print(f"PASS - {msg}")
+        else:
+            failed += 1
+            print(f"FAIL - {msg}")
+
+    print("=" * 70)
+    print(f"Results: {passed} passed, {failed} failed out of {len(specs)}")
+
+    if args.output:
+        output_path = Path(args.output)
+        output_path.write_text(json.dumps(results, indent=2) + "\n")
+        print(f"Detailed results written to {output_path}")
+
+    if failed > 0:
+        print("\nFailed models:")
+        for r in results:
+            if not r["passed"]:
+                print(f"  {r['model_name']}: {r['message']}")
+        sys.exit(1)
+    else:
+        print("\nAll models passed smoke test!")
+
+
+if __name__ == "__main__":
+    main()

scripts/smoke_test_submission.py

@@ -0,0 +1,266 @@
+#!/usr/bin/env python3
+"""End-to-end smoke test mimicking a blue team submission via Modal.
+
+Picks 20 models from the blue team registry and 50 images from the
+stimuli catalog, runs the full extraction + CKA scoring pipeline
+on Modal, and verifies the results.
+
+Usage:
+    python scripts/smoke_test_submission.py \
+        --registry configs/blue_team_model_registry.json \
+        --stimuli configs/blue_team_images.jsonl
+
+    # With custom counts:
+    python scripts/smoke_test_submission.py \
+        --num-models 5 --num-stimuli 10
+"""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import sys
+from pathlib import Path
+
+
+def load_registry(path: str) -> list[dict]:
+    data = json.loads(Path(path).read_text())
+    if isinstance(data, dict):
+        return data["models"]
+    return data
+
+
+def load_stimuli(path: str) -> list[dict]:
+    p = Path(path)
+    if p.suffix == ".jsonl":
+        return [json.loads(line) for line in p.read_text().splitlines() if line.strip()]
+    data = json.loads(p.read_text())
+    if isinstance(data, dict):
+        return data["stimuli"]
+    return data
+
+
+def cache_key_from_payload(registry: list[dict], stimuli: list[dict]) -> str:
+    payload = {"registry": registry, "stimuli": stimuli}
+    encoded = json.dumps(payload, sort_keys=True).encode("utf-8")
+    digest = hashlib.sha1(encoded).hexdigest()[:12]
+    return f"smoke_{digest}"
+
+
+def main():
+    parser = argparse.ArgumentParser(description="End-to-end blue team submission smoke test on Modal")
+    parser.add_argument("--registry", default="configs/blue_team_model_registry.json",
+                        help="Path to blue team model registry JSON")
+    parser.add_argument("--stimuli", default="configs/blue_team_images.jsonl",
+                        help="Path to stimuli catalog JSONL")
+    parser.add_argument("--num-models", type=int, default=20,
+                        help="Number of models to select (default: 20)")
+    parser.add_argument("--num-stimuli", type=int, default=50,
+                        help="Number of stimuli to select (default: 50)")
+    parser.add_argument("--app-name", default="iclr2026-eval",
+                        help="Modal app name")
+    parser.add_argument("--batch-size", type=int, default=64,
+                        help="Batch size for embedding extraction")
+    args = parser.parse_args()
+
+    # Load full registry and stimuli
+    full_registry = load_registry(args.registry)
+    full_stimuli = load_stimuli(args.stimuli)
+
+    print(f"Full registry: {len(full_registry)} models")
+    print(f"Full stimuli:  {len(full_stimuli)} images")
+
+    # Select subset - pick models evenly spaced for diversity
+    n_models = min(args.num_models, len(full_registry))
+    if n_models < len(full_registry):
+        step = len(full_registry) / n_models
+        indices = [int(i * step) for i in range(n_models)]
+        subset_registry = [full_registry[i] for i in indices]
+    else:
+        subset_registry = full_registry
+
+    n_stimuli = min(args.num_stimuli, len(full_stimuli))
+    subset_stimuli = full_stimuli[:n_stimuli]
+
+    model_names = [m["model_name"] for m in subset_registry]
+    print(f"\nSelected {len(subset_registry)} models:")
+    for m in model_names:
+        print(f"  - {m}")
+    print(f"\nSelected {len(subset_stimuli)} stimuli (first {n_stimuli} from catalog)")
+
+    # Build cache key
+    cache_key = cache_key_from_payload(subset_registry, subset_stimuli)
+    print(f"\nCache key: {cache_key}")
+
+    # Connect to Modal
+    print("\nConnecting to Modal...")
+    import modal
+
+    extract_fn = modal.Function.from_name(args.app_name, "extract_embeddings_s3")
+    score_fn = modal.Function.from_name(args.app_name, "compute_pairwise_cka")
+    print("Modal connection OK")
+
+    # Step 1: Extract embeddings
+    print(f"\n{'=' * 60}")
+    print("STEP 1: Extracting embeddings...")
+    print(f"{'=' * 60}")
+
+    extract_result = extract_fn.remote(
+        model_registry=subset_registry,
+        stimuli=subset_stimuli,
+        cache_key=cache_key,
+        batch_size=args.batch_size,
+        reuse_cache=False,  # Force fresh extraction for smoke test
+    )
+
+    print(f"\nExtraction result:")
+    print(f"  cache_key:   {extract_result.get('cache_key')}")
+    print(f"  num_stimuli: {extract_result.get('num_stimuli')}")
+    print(f"  models:      {len(extract_result.get('models', []))}")
+
+    for m in extract_result.get("models", []):
+        print(
+            f"    {m['model_name']:45s} layer={m.get('layer', '?'):25s} "
+            f"dim={m.get('dim', '?'):>6}  samples={m.get('num_samples', '?')}"
+        )
+
+    # Validate extraction
+    errors = []
+    for m in extract_result.get("models", []):
+        if m.get("num_samples") != n_stimuli:
+            errors.append(f"  {m['model_name']}: expected {n_stimuli} samples, got {m.get('num_samples')}")
+        if m.get("dim", 0) <= 0:
+            errors.append(f"  {m['model_name']}: invalid dim {m.get('dim')}")
+
+    if errors:
+        print(f"\nExtraction ERRORS:")
+        for e in errors:
+            print(e)
+        sys.exit(1)
+    print(f"\nExtraction: ALL {len(extract_result.get('models', []))} models OK")
+
+    # Step 2: Compute pairwise CKA
+    print(f"\n{'=' * 60}")
+    print("STEP 2: Computing pairwise CKA...")
+    print(f"{'=' * 60}")
+
+    cka_result = score_fn.remote(
+        cache_key=cache_key,
+        model_names=model_names,
+    )
+
+    avg_cka = cka_result.get("avg_cka", 0.0)
+    pairwise = cka_result.get("pairwise", [])
+
+    print(f"\nCKA results:")
+    print(f"  avg_cka:     {avg_cka:.6f}")
+    print(f"  num_pairs:   {len(pairwise)}")
+
+    expected_pairs = n_models * (n_models - 1) // 2
+    if len(pairwise) != expected_pairs:
+        print(f"  WARNING: expected {expected_pairs} pairs, got {len(pairwise)}")
+
+    # Show top 5 and bottom 5 pairs
+    sorted_pairs = sorted(pairwise, key=lambda x: x.get("cka", 0), reverse=True)
+    print(f"\n  Top 5 most similar pairs:")
+    for p in sorted_pairs[:5]:
+        ma = p.get("model_a", "?")
+        mb = p.get("model_b", "?")
+        cka = p.get("cka", 0.0)
+        print(f"    {ma:40s} <-> {mb:40s}  CKA={cka:.6f}")
+
+    print(f"\n  Bottom 5 least similar pairs:")
+    for p in sorted_pairs[-5:]:
+        ma = p.get("model_a", "?")
+        mb = p.get("model_b", "?")
+        cka = p.get("cka", 0.0)
+        print(f"    {ma:40s} <-> {mb:40s}  CKA={cka:.6f}")
+
+    # Validate CKA results
+    cka_errors = []
+    for p in pairwise:
+        cka_val = p.get("cka")
+        if cka_val is None:
+            cka_errors.append("  Missing CKA value for a pair")
+        elif not (-0.1 <= cka_val <= 1.5):  # Allow slight numerical overshoot
+            ma = p.get("model_a", "?")
+            mb = p.get("model_b", "?")
+            cka_errors.append(f"  {ma} <-> {mb}: CKA={cka_val} out of expected range")
+
+    if cka_errors:
+        print(f"\nCKA ERRORS:")
+        for e in cka_errors:
+            print(e)
+        sys.exit(1)
+
+    # Step 3: Validate submission format (local check)
+    print(f"\n{'=' * 60}")
+    print("STEP 3: Validating submission format...")
+    print(f"{'=' * 60}")
+
+    submission_payload = {
+        "models": [
+            {"model_name": m["model_name"], "layer_name": m["layer"]}
+            for m in subset_registry
+        ]
+    }
+    print(f"\n  Sample submission entry: {json.dumps(submission_payload['models'][0])}")
+
+    try:
+        from src.hackathon.validation import (
+            BLUE_TEAM_REQUIRED_MODELS,
+            load_model_registry,
+            load_model_registry_specs,
+            validate_blue_submission,
+        )
+
+        registry_names = load_model_registry(args.registry)
+        registry_specs = load_model_registry_specs(args.registry)
+
+        if n_models == BLUE_TEAM_REQUIRED_MODELS:
+            validated_names = validate_blue_submission(
+                submission_payload,
+                model_registry=registry_names,
+                registry_specs=registry_specs,
+            )
+            print(f"  Validation OK: {len(validated_names)} models accepted")
+        else:
+            print(f"  Skipping count validation (selected {n_models}, required {BLUE_TEAM_REQUIRED_MODELS})")
+            # Still check layer matching
+            mismatches = []
+            for m in subset_registry:
+                name = m["model_name"]
+                if name in registry_specs:
+                    expected = registry_specs[name]["layer"]
+                    submitted = m["layer"]
+                    if submitted != expected:
+                        mismatches.append(f"    {name}: layer={submitted} expected={expected}")
+            if mismatches:
+                print(f"  Layer MISMATCHES:")
+                for mm in mismatches:
+                    print(mm)
+                sys.exit(1)
+            print(f"  Layer validation OK for all {n_models} models")
+    except ImportError as exc:
+        print(f"  Skipping validation (import failed: {exc})")
+    except Exception as exc:
+        print(f"  Validation FAILED: {exc}")
+        sys.exit(1)
+
+    # Summary
+    blue_score = avg_cka
+    print(f"\n{'=' * 60}")
+    print("SMOKE TEST PASSED")
+    print(f"{'=' * 60}")
+    print(f"  Models:       {n_models}")
+    print(f"  Stimuli:      {n_stimuli}")
+    print(f"  Avg CKA:      {avg_cka:.6f}")
+    print(f"  Blue score:   {blue_score:.6f}")
+    print(f"  Pairs:        {len(pairwise)} / {expected_pairs} expected")
+    print(f"  All layers:   validated against registry")
+    print()
+
+
+if __name__ == "__main__":
+    main()

scripts/submit_blue_hf_dataset.py

@@ -0,0 +1,36 @@
+import os
+import json
+import argparse
+
+from datasets import Dataset
+from dotenv import load_dotenv
+load_dotenv()
+
+DATASET_NAME = "bkhmsi/test-realign-hackathon-blue-team"
+TOKEN_ENV = "HF_TOKEN"
+
+def push_to_hf(token: str, stimuli: list[dict[str, str]]) -> None:
+    dataset = Dataset.from_list(stimuli)
+    print(f"> Pushing {len(dataset)} records to HF dataset {DATASET_NAME}")
+    dataset.push_to_hub(DATASET_NAME, token=token, private=True)
+
+def read_jsonl(path: str) -> list[str]:
+    with open(path, "r") as f:
+        data = [json.loads(line) for line in f]
+    return data
+
+def main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--path", type=str, required=False, default="test_submissions/blue_submission.jsonl", help="Path to the JSON file containing model names.")
+    args = parser.parse_args()
+    path = args.path
+
+    token = os.environ.get(TOKEN_ENV, "").strip()
+    if not token:
+        raise ValueError("Set HF_TOKEN before running this script.")
+
+    stimuli = read_jsonl(path)
+    push_to_hf(token, stimuli)
+
+if __name__ == "__main__":
+    main()

scripts/submit_red_hf_dataset.py

@@ -0,0 +1,36 @@
+import os
+import json
+import argparse
+
+from datasets import Dataset
+from dotenv import load_dotenv
+load_dotenv()
+
+DATASET_NAME = "bkhmsi/test-realign-hackathon-red-team"
+TOKEN_ENV = "HF_TOKEN"
+
+def push_to_hf(token: str, stimuli: list[dict[str, str]]) -> None:
+    dataset = Dataset.from_list(stimuli)
+    print(f"> Pushing {len(dataset)} records to HF dataset {DATASET_NAME}")
+    dataset.push_to_hub(DATASET_NAME, token=token, private=True)
+
+def read_jsonl(path: str) -> list[str]:
+    with open(path, "r") as f:
+        data = [json.loads(line) for line in f]
+    return data
+
+def main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--path", type=str, required=False, default="test_submissions/red_submission.jsonl", help="Path to the JSON file containing stimuli names.")
+    args = parser.parse_args()
+    path = args.path
+
+    token = os.environ.get(TOKEN_ENV, "").strip()
+    if not token:
+        raise ValueError("Set HF_TOKEN before running this script.")
+
+    stimuli = read_jsonl(path)
+    push_to_hf(token, stimuli)
+
+if __name__ == "__main__":
+    main()

scripts/validate_submission.py

@@ -0,0 +1,71 @@
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import sys
+from pathlib import Path
+
+from src.hackathon.validation import (
+    MODEL_REGISTRY_ENV,
+    STIMULI_CATALOG_ENV,
+    load_model_registry,
+    load_stimuli_catalog,
+    validate_blue_submission,
+    validate_red_submission,
+)
+
+
+def _load_payload(path: str) -> dict:
+    if path == "-":
+        raw = sys.stdin.read()
+        source = "stdin"
+    else:
+        source = path
+        try:
+            raw = Path(path).read_text()
+        except FileNotFoundError as exc:
+            raise ValueError(f"File not found: {path}") from exc
+
+    try:
+        data = json.loads(raw)
+    except json.JSONDecodeError as exc:
+        raise ValueError(f"Invalid JSON in {source}: {exc}") from exc
+
+    if not isinstance(data, dict):
+        raise ValueError("Submission JSON must be an object.")
+    return data
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description="Validate blue/red submission JSON against the contract.")
+    parser.add_argument("payload", help="Path to submission JSON or '-' for stdin.")
+    parser.add_argument("--team", choices=["blue", "red"], required=True, help="Submission team to validate.")
+    parser.add_argument(
+        "--model-registry",
+        default=os.environ.get(MODEL_REGISTRY_ENV, ""),
+        help=f"Model registry JSON path (defaults to ${MODEL_REGISTRY_ENV}).",
+    )
+    parser.add_argument(
+        "--stimuli-catalog",
+        default=os.environ.get(STIMULI_CATALOG_ENV, ""),
+        help=f"Stimuli catalog JSON/JSONL path (defaults to ${STIMULI_CATALOG_ENV}).",
+    )
+
+    args = parser.parse_args()
+    payload = _load_payload(args.payload)
+
+    if args.team == "blue":
+        registry = load_model_registry(args.model_registry or None)
+        models = validate_blue_submission(payload, model_registry=registry)
+        print(f"OK: blue submission validated ({len(models)} models).")
+    else:
+        catalog = load_stimuli_catalog(args.stimuli_catalog or None)
+        stimuli = validate_red_submission(payload, stimuli_catalog=catalog)
+        print(f"OK: red submission validated ({len(stimuli)} stimuli).")
+
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/verify_cka.py

@@ -0,0 +1,55 @@
+import numpy as np
+
+from src.cka.compute import linear_cka, linear_cka_feature
+
+
+def hsic_biased(k, l):
+    h = np.eye(k.shape[0], dtype=k.dtype) - 1 / k.shape[0]
+    return float(np.trace(k @ h @ l @ h))
+
+
+def hsic_unbiased(k, l):
+    m = k.shape[0]
+    k_tilde = k.copy()
+    l_tilde = l.copy()
+    np.fill_diagonal(k_tilde, 0.0)
+    np.fill_diagonal(l_tilde, 0.0)
+    hsic_value = (
+        (np.sum(k_tilde * l_tilde.T))
+        + (np.sum(k_tilde) * np.sum(l_tilde) / ((m - 1) * (m - 2)))
+        - (2 * np.sum(k_tilde @ l_tilde) / (m - 2))
+    )
+    hsic_value /= m * (m - 3)
+    return float(hsic_value)
+
+
+def ref_cka(a, b, unbiased=False):
+    k = a @ a.T
+    l = b @ b.T
+    hsic_fn = hsic_unbiased if unbiased else hsic_biased
+    hsic_kk = hsic_fn(k, k)
+    hsic_ll = hsic_fn(l, l)
+    hsic_kl = hsic_fn(k, l)
+    return float(hsic_kl / (np.sqrt(hsic_kk * hsic_ll) + 1e-6))
+
+
+def main():
+    rng = np.random.default_rng(0)
+    a = rng.standard_normal((64, 128)).astype(np.float64)
+    b = rng.standard_normal((64, 128)).astype(np.float64)
+
+    our_biased = linear_cka(a, b, unbiased=False)
+    ref_biased = ref_cka(a, b, unbiased=False)
+
+    our_unbiased = linear_cka(a, b, unbiased=True)
+    ref_unbiased = ref_cka(a, b, unbiased=True)
+
+    our_feature = linear_cka_feature(a, b)
+
+    print("biased:", our_biased, ref_biased, "diff", abs(our_biased - ref_biased))
+    print("unbiased:", our_unbiased, ref_unbiased, "diff", abs(our_unbiased - ref_unbiased))
+    print("feature:", our_feature, ref_biased, "diff", abs(our_feature - ref_biased))
+
+
+if __name__ == "__main__":
+    main()

slurm-verify-8056978.out

@@ -0,0 +1,2 @@
+ok
+

src/about.py

@@ -0,0 +1,153 @@
+TITLE = """
+<div align="center" class="hackathon-header">
+    <h1 id="space-title">The Re-Align Hackathon</h1>
+    <p class="hackathon-subtitle">ICLR 2026 Workshop on Representational Alignment</p>
+    <h4 class="hackathon-nav">
+        <a href="#blue-team">Blue Team</a>
+        <span class="nav-divider">&bull;</span>
+        <a href="#red-team">Red Team</a>
+        <span class="nav-divider">&bull;</span>
+        <a href="#getting-started">Getting Started</a>
+        <span class="nav-divider">&bull;</span>
+        <a href="https://representational-alignment.github.io/2026/">Workshop</a>
+    </h4>
+</div>
+"""
+
+INTRODUCTION_TEXT = """
+## Challenge Overview
+
+The Re-Align Hackathon explores **representational alignment** across vision models.
+
+- **Blue Team** — Find 20 models with the most aligned representations (maximize CKA)
+- **Red Team** — Find stimuli that drive model representations apart (minimize CKA)
+
+Submissions are scored using [CKA (Centered Kernel Alignment)](https://arxiv.org/abs/1905.00414) computed on embeddings extracted from a fixed set of held-out images.
+"""
+
+BLUE_SUBMISSION_TEXT = """
+### Blue Team — Maximize Alignment
+
+Select **20 models** from the registry whose representations are most aligned.
+Your score is the **mean pairwise CKA** across your selected models — higher is better.
+
+**How to submit:**
+1. Create a HuggingFace dataset with `model_name` and `layer_name` columns
+2. Paste the dataset link below and click **Generate JSON**
+3. Review the generated JSON, then click **Submit**
+"""
+
+RED_SUBMISSION_TEXT = """
+### Red Team — Maximize Divergence
+
+Select **1000 stimuli** images that cause the fixed set of models to produce **divergent representations**.
+Your score is **1 − avg CKA** — higher divergence ranks higher.
+
+**How to submit:**
+1. Create a HuggingFace dataset with `dataset_name` and `image_identifier` columns
+2. Paste the dataset link below and click **Generate JSON**
+3. Review the generated JSON, then click **Submit**
+"""
+
+
+
+GETTING_STARTED_TEXT = """
+## Getting Started
+
+To participate in the hackathon, you can submit model selections (Blue Team) or stimulus selections
+(Red Team). Each submission will be scored and ranked on the respective leaderboard.
+
+### Hugging Face datasets (private only)
+
+You can optionally host the model/stimulus selections as a Hugging Face Dataset and paste the
+dataset link into the app. These datasets must be private (do not publish public submissions).
+For end-to-end examples, refer to `scripts/submit_blue_hf_dataset.py` and
+`scripts/submit_red_hf_dataset.py`.
+
+### Blue Team dataset
+
+Expected columns:
+- `model_name`: string model name
+- `layer_name`: string layer name
+
+Example dataset row:
+```json
+{
+  "model_name": "resnet18",
+  "layer_name": "fc"
+}
+```
+
+Create the dataset (example with `datasets` + `huggingface_hub`):
+```bash
+pip install datasets huggingface_hub
+```
+```py
+from datasets import Dataset
+from huggingface_hub import login
+
+login(token="hf_...")  # optional if you already ran `huggingface-cli login`
+
+rows = [
+    {"model_name": "resnet18", "layer_name": "fc"},
+    {"model_name": "resnet34", "layer_name": "fc"},
+]
+ds = Dataset.from_list(rows)
+ds.push_to_hub("your-username/blue-team-submission", private=True)
+```
+
+Paste the dataset link into the app:
+```
+your-username/blue-team-submission
+```
+
+### Red Team dataset
+
+Expected columns:
+- `dataset_name`: string dataset name
+- `image_identifier`: string image identifier within the dataset
+
+Example dataset row:
+```json
+{
+  "dataset_name": "imagenet_val",
+  "image_identifier": "ILSVRC2012_val_00000001.JPEG"
+}
+```
+
+Create the dataset:
+```bash
+pip install datasets huggingface_hub
+```
+```py
+from datasets import Dataset
+from huggingface_hub import login
+
+login(token="hf_...")  # optional if you already ran `huggingface-cli login`
+
+rows = [
+    {
+        "dataset_name": "imagenet_val",
+        "image_identifier": "ILSVRC2012_val_00000001.JPEG",
+    },
+    {
+        "dataset_name": "objectnet",
+        "image_identifier": "objectnet-1.0/images/baking_sheet/01f362ca48a547c.png",
+    },
+]
+ds = Dataset.from_list(rows)
+ds.push_to_hub("your-username/red-team-submission", private=True)
+```
+
+Paste the dataset link into the app:
+```
+your-username/red-team-submission
+```
+
+### Access token (for private datasets)
+
+1. Go to your Hugging Face account settings: `https://huggingface.co/settings/tokens`
+2. Create a new token with the `Read` scope.
+3. Copy the token (it starts with `hf_...`).
+4. Paste it into the "HuggingFace access token (optional)" textbox in the app.
+"""

src/cka/__init__.py

@@ -0,0 +1,11 @@
+from src.cka.compute import linear_cka
+from src.cka.embeddings import get_cached_embeddings, list_cached_embedding_names
+from src.cka.storage import append_submission, load_submissions
+
+__all__ = [
+    "append_submission",
+    "get_cached_embeddings",
+    "linear_cka",
+    "list_cached_embedding_names",
+    "load_submissions",
+]

src/cka/compute.py

@@ -0,0 +1,75 @@
+from __future__ import annotations
+
+import numpy as np
+
+
+def _validate_inputs(x: np.ndarray, y: np.ndarray) -> tuple[np.ndarray, np.ndarray]:
+    x = np.asarray(x, dtype=np.float64)
+    y = np.asarray(y, dtype=np.float64)
+
+    if x.ndim != 2 or y.ndim != 2:
+        raise ValueError("CKA expects 2D arrays shaped [num_samples, dim].")
+    if x.shape[0] != y.shape[0]:
+        raise ValueError("CKA expects the same number of samples in both embeddings.")
+
+    return x, y
+
+
+def hsic_biased(k: np.ndarray, l: np.ndarray) -> float:
+    """Biased HSIC, matches the reference implementation."""
+    m = k.shape[0]
+    h = np.eye(m, dtype=k.dtype) - (1.0 / m)
+    return float(np.trace(k @ h @ l @ h))
+
+
+def hsic_unbiased(k: np.ndarray, l: np.ndarray) -> float:
+    """Unbiased HSIC as in Song et al. (2012)."""
+    m = k.shape[0]
+    if m < 4:
+        return 0.0
+
+    k_tilde = k.copy()
+    l_tilde = l.copy()
+    np.fill_diagonal(k_tilde, 0.0)
+    np.fill_diagonal(l_tilde, 0.0)
+
+    term1 = np.sum(k_tilde * l_tilde.T)
+    term2 = (np.sum(k_tilde) * np.sum(l_tilde)) / ((m - 1) * (m - 2))
+    term3 = (2 * np.sum(k_tilde @ l_tilde)) / (m - 2)
+    return float((term1 + term2 - term3) / (m * (m - 3)))
+
+
+def linear_cka_feature(x: np.ndarray, y: np.ndarray, eps: float = 1e-6) -> float:
+    """Direct feature-space linear CKA (equivalent to biased HSIC for linear kernels)."""
+    x, y = _validate_inputs(x, y)
+
+    x = x - x.mean(axis=0, keepdims=True)
+    y = y - y.mean(axis=0, keepdims=True)
+
+    numerator = np.linalg.norm(x.T @ y, ord="fro") ** 2
+    denom = np.linalg.norm(x.T @ x, ord="fro") * np.linalg.norm(y.T @ y, ord="fro")
+    if denom == 0:
+        return 0.0
+    return float(numerator / (denom + eps))
+
+
+def linear_cka(
+    x: np.ndarray,
+    y: np.ndarray,
+    *,
+    unbiased: bool = False,
+    eps: float = 1e-6,
+) -> float:
+    """Linear CKA computed via HSIC, matching the reference implementation."""
+    x, y = _validate_inputs(x, y)
+
+    k = x @ x.T
+    l = y @ y.T
+
+    hsic_fn = hsic_unbiased if unbiased else hsic_biased
+    hsic_kk = hsic_fn(k, k)
+    hsic_ll = hsic_fn(l, l)
+    hsic_kl = hsic_fn(k, l)
+
+    denom = np.sqrt(hsic_kk * hsic_ll)
+    return float(hsic_kl / (denom + eps))

src/cka/embeddings.py

@@ -0,0 +1,29 @@
+from __future__ import annotations
+
+from functools import lru_cache
+
+import numpy as np
+
+
+@lru_cache(maxsize=1)
+def get_cached_embeddings() -> dict[str, np.ndarray]:
+    """Return dummy cached embeddings until real data is wired in."""
+    rng = np.random.default_rng(1234)
+    num_samples = 128
+    dim = 64
+
+    base = rng.normal(size=(num_samples, dim)).astype(np.float32)
+    embeddings = {
+        "base": base,
+        "base_noise_small": (base + 0.05 * rng.normal(size=(num_samples, dim))).astype(np.float32),
+        "base_noise_medium": (base + 0.2 * rng.normal(size=(num_samples, dim))).astype(np.float32),
+        "random_1": rng.normal(size=(num_samples, dim)).astype(np.float32),
+        "random_2": rng.normal(size=(num_samples, dim)).astype(np.float32),
+        "random_3": rng.normal(size=(num_samples, dim)).astype(np.float32),
+    }
+
+    return embeddings
+
+
+def list_cached_embedding_names() -> list[str]:
+    return sorted(get_cached_embeddings().keys())

src/cka/storage.py

@@ -0,0 +1,46 @@
+from __future__ import annotations
+
+import json
+import os
+import threading
+from typing import Any
+
+DEFAULT_DATA_PATH = os.environ.get(
+    "CKA_DATA_PATH",
+    os.path.join(os.getcwd(), "cka-data", "submissions.json"),
+)
+
+_LOCK = threading.Lock()
+
+
+def load_submissions(data_path: str = DEFAULT_DATA_PATH) -> list[dict[str, Any]]:
+    if not os.path.exists(data_path):
+        return []
+
+    with _LOCK:
+        with open(data_path, "r") as f:
+            data = json.load(f)
+
+    if not isinstance(data, list):
+        raise ValueError(f"Expected a list in {data_path}.")
+
+    return data
+
+
+def save_submissions(submissions: list[dict[str, Any]], data_path: str = DEFAULT_DATA_PATH) -> None:
+    data_dir = os.path.dirname(data_path)
+    if data_dir:
+        os.makedirs(data_dir, exist_ok=True)
+
+    tmp_path = f"{data_path}.tmp"
+    with _LOCK:
+        with open(tmp_path, "w") as f:
+            json.dump(submissions, f, indent=2)
+        os.replace(tmp_path, data_path)
+
+
+def append_submission(submission: dict[str, Any], data_path: str = DEFAULT_DATA_PATH) -> list[dict[str, Any]]:
+    submissions = load_submissions(data_path)
+    submissions.append(submission)
+    save_submissions(submissions, data_path)
+    return submissions

src/display/css_html_js.py

@@ -0,0 +1,243 @@
+custom_css = """
+
+@import url("https://fonts.googleapis.com/css2?family=Space+Grotesk:wght@400;600;700&family=IBM+Plex+Mono:wght@400;600&display=swap");
+
+/* Force dark theme everywhere */
+:root, .dark, .light {
+    color-scheme: dark !important;
+    --bg-1: #0d1117;
+    --bg-2: #161b22;
+    --card-bg: #1c2128;
+    --ink-1: #e6edf3;
+    --ink-2: #8b949e;
+    --accent: #58a6ff;
+    --accent-hover: #79b8ff;
+    --accent-2: #bc8cff;
+    --border: #30363d;
+    --link-text-color: var(--accent);
+    --blue-team: #58a6ff;
+    --blue-team-hover: #79b8ff;
+    --red-team: #f85149;
+    --red-team-hover: #ff7b72;
+    --success: #3fb950;
+    --error: #f85149;
+}
+
+body,
+.gradio-container {
+    font-family: "Space Grotesk", "Trebuchet MS", sans-serif;
+    color: var(--ink-1) !important;
+    background: var(--bg-1) !important;
+}
+
+.gradio-container .block {
+    background: var(--card-bg) !important;
+    border: 1px solid var(--border) !important;
+    border-radius: 10px;
+    box-shadow: 0 4px 12px rgba(0, 0, 0, 0.3);
+}
+
+/* Force text color on all Gradio elements */
+.gradio-container *,
+.gradio-container .prose,
+.gradio-container .prose p,
+.gradio-container .prose li,
+.gradio-container .prose h1,
+.gradio-container .prose h2,
+.gradio-container .prose h3,
+.gradio-container .prose h4,
+.gradio-container .prose strong,
+.gradio-container label,
+.gradio-container span {
+    color: var(--ink-1);
+}
+
+.gradio-container .prose a,
+.gradio-container a {
+    color: var(--accent) !important;
+}
+
+.gradio-container .prose code,
+.gradio-container code {
+    background: #282e36 !important;
+    color: #e6edf3 !important;
+}
+
+.gradio-container .prose pre,
+.gradio-container pre {
+    background: #161b22 !important;
+    border: 1px solid var(--border) !important;
+}
+
+.gradio-container input,
+.gradio-container textarea,
+.gradio-container select,
+.gr-text-input textarea,
+.gr-text-input input {
+    color: var(--ink-1) !important;
+    background: #0d1117 !important;
+    border: 1px solid var(--border) !important;
+}
+
+.gradio-container input::placeholder,
+.gradio-container textarea::placeholder {
+    color: var(--ink-2) !important;
+}
+
+/* Tabs */
+.gradio-container .tabs {
+    background: transparent !important;
+}
+
+.gradio-container button.tab-nav,
+.gradio-container .tab-nav button {
+    color: var(--ink-2) !important;
+    background: transparent !important;
+}
+
+.gradio-container button.tab-nav.selected,
+.gradio-container .tab-nav button.selected {
+    color: var(--ink-1) !important;
+    border-bottom-color: var(--accent) !important;
+}
+
+/* Accordion */
+.gradio-container .accordion {
+    background: var(--card-bg) !important;
+    border-color: var(--border) !important;
+}
+
+/* Dataframe / table */
+.gradio-container table {
+    background: var(--card-bg) !important;
+}
+
+.gradio-container table th {
+    background: #21262d !important;
+    color: var(--ink-1) !important;
+    border-color: var(--border) !important;
+}
+
+.gradio-container table td {
+    background: var(--card-bg) !important;
+    color: var(--ink-1) !important;
+    border-color: var(--border) !important;
+}
+
+.gradio-container table tr:nth-child(even) td {
+    background: #161b22 !important;
+}
+
+/* Buttons */
+.gr-button {
+    background: var(--accent) !important;
+    color: #ffffff !important;
+    border: 1px solid transparent !important;
+    font-weight: 600;
+    letter-spacing: 0.01em;
+}
+
+.gr-button:hover {
+    background: var(--accent-hover) !important;
+}
+
+.gr-button.secondary {
+    background: var(--card-bg) !important;
+    color: var(--ink-1) !important;
+    border: 1px solid var(--border) !important;
+}
+
+.gr-button.secondary:hover {
+    border-color: var(--accent) !important;
+    color: var(--accent) !important;
+}
+
+/* Header */
+.hackathon-header {
+    margin: 10px 0 24px;
+}
+
+.hackathon-header h1 {
+    font-size: 40px;
+    letter-spacing: 0.02em;
+    margin-bottom: 6px;
+    color: var(--ink-1) !important;
+}
+
+.hackathon-subtitle {
+    color: var(--ink-2) !important;
+    font-size: 18px;
+    margin-top: -4px;
+    margin-bottom: 8px;
+}
+
+.hackathon-nav {
+    font-weight: 500;
+    margin: 4px 0 10px;
+}
+
+.hackathon-nav a {
+    color: var(--ink-2) !important;
+    text-decoration: none;
+    padding: 0 8px;
+}
+
+.hackathon-nav a:hover {
+    color: var(--accent) !important;
+}
+
+.nav-divider {
+    color: var(--accent-2) !important;
+}
+
+.markdown-text {
+    font-size: 16px !important;
+}
+
+.tab-buttons button {
+    font-size: 20px;
+}
+
+/* Blue team accent */
+.blue-team .gr-button.primary,
+.blue-team .gr-button:not(.secondary) {
+    background: var(--blue-team) !important;
+}
+.blue-team .gr-button:not(.secondary):hover {
+    background: var(--blue-team-hover) !important;
+}
+.blue-team-header {
+    color: var(--blue-team) !important;
+    border-left: 4px solid var(--blue-team);
+    padding-left: 12px;
+    margin-bottom: 12px;
+}
+
+/* Red team accent */
+.red-team .gr-button.primary,
+.red-team .gr-button:not(.secondary) {
+    background: var(--red-team) !important;
+}
+.red-team .gr-button:not(.secondary):hover {
+    background: var(--red-team-hover) !important;
+}
+.red-team-header {
+    color: var(--red-team) !important;
+    border-left: 4px solid var(--red-team);
+    padding-left: 12px;
+    margin-bottom: 12px;
+}
+
+/* Status message styles */
+.status-error { color: var(--error) !important; font-size: 16px; text-align: center; font-weight: 600; }
+.status-warning { color: var(--accent-2) !important; font-size: 16px; text-align: center; font-weight: 600; }
+.status-success { color: var(--success) !important; font-size: 16px; text-align: center; font-weight: 600; }
+"""
+
+get_window_url_params = """
+    function(url_params) {
+        const params = new URLSearchParams(window.location.search);
+        url_params = Object.fromEntries(params);
+        return url_params;
+    }
+    """

src/display/formatting.py

@@ -0,0 +1,27 @@
+def model_hyperlink(link, model_name):
+    return f'<a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{model_name}</a>'
+
+
+def make_clickable_model(model_name):
+    link = f"https://huggingface.co/{model_name}"
+    return model_hyperlink(link, model_name)
+
+
+def styled_error(error):
+    return f"<p class='status-error'>{error}</p>"
+
+
+def styled_warning(warn):
+    return f"<p class='status-warning'>{warn}</p>"
+
+
+def styled_message(message):
+    return f"<p class='status-success'>{message}</p>"
+
+
+def has_no_nan_values(df, columns):
+    return df[columns].notna().all(axis=1)
+
+
+def has_nan_values(df, columns):
+    return df[columns].isna().any(axis=1)

src/hackathon/__init__.py

@@ -0,0 +1,17 @@
+from src.hackathon.data import get_dummy_model_embeddings, list_dummy_models, list_dummy_stimuli, stimulus_key
+from src.hackathon.scoring import score_blue, score_blue_with_pairwise, score_red, score_red_with_pairwise
+from src.hackathon.storage import append_submission, load_submissions, save_submissions
+
+__all__ = [
+    "append_submission",
+    "get_dummy_model_embeddings",
+    "list_dummy_models",
+    "list_dummy_stimuli",
+    "load_submissions",
+    "save_submissions",
+    "score_blue",
+    "score_blue_with_pairwise",
+    "score_red",
+    "score_red_with_pairwise",
+    "stimulus_key",
+]

src/hackathon/data.py

@@ -0,0 +1,121 @@
+from __future__ import annotations
+
+from functools import lru_cache
+from typing import Iterable, Mapping
+
+import numpy as np
+
+Stimulus = Mapping[str, str]
+
+_DUMMY_MODELS = [
+    "vit_base_patch16_224",
+    "vit_large_patch16_224",
+    "resnet50",
+    "resnet101",
+    "convnext_base",
+    "convnext_large",
+    "deit_base_patch16_224",
+    "clip_vit_b32",
+    "swin_tiny_patch4_window7_224",
+]
+
+_DUMMY_STIMULI = [
+    {"dataset_name": "cifar100", "image_identifier": "test/airplane/image_0001.png"},
+    {"dataset_name": "cifar100", "image_identifier": "test/bear/image_0007.png"},
+    {"dataset_name": "cifar100", "image_identifier": "test/bottle/image_0012.png"},
+    {"dataset_name": "cifar100", "image_identifier": "test/bus/image_0021.png"},
+    {"dataset_name": "cifar100", "image_identifier": "test/girl/image_0033.png"},
+    {"dataset_name": "cifar100", "image_identifier": "test/keyboard/image_0044.png"},
+    {"dataset_name": "cifar100", "image_identifier": "test/rocket/image_0051.png"},
+    {"dataset_name": "cifar100", "image_identifier": "test/whale/image_0068.png"},
+    {"dataset_name": "imagenet1k", "image_identifier": "val/n01440764/ILSVRC2012_val_00000964.JPEG"},
+    {"dataset_name": "imagenet1k", "image_identifier": "val/n02123159/ILSVRC2012_val_00001459.JPEG"},
+    {"dataset_name": "imagenet1k", "image_identifier": "val/n03255030/ILSVRC2012_val_00001903.JPEG"},
+    {"dataset_name": "imagenet1k", "image_identifier": "val/n03445777/ILSVRC2012_val_00003572.JPEG"},
+    {"dataset_name": "imagenet1k", "image_identifier": "val/n03729826/ILSVRC2012_val_00005336.JPEG"},
+    {"dataset_name": "imagenet1k", "image_identifier": "val/n03902125/ILSVRC2012_val_00006614.JPEG"},
+    {"dataset_name": "imagenet1k", "image_identifier": "val/n04254777/ILSVRC2012_val_00007190.JPEG"},
+    {"dataset_name": "imagenet1k", "image_identifier": "val/n04557648/ILSVRC2012_val_00009024.JPEG"},
+]
+
+
+def list_dummy_models() -> list[str]:
+    return list(_DUMMY_MODELS)
+
+
+def list_dummy_stimuli() -> list[dict[str, str]]:
+    return [dict(item) for item in _DUMMY_STIMULI]
+
+
+def stimulus_key(stimulus: Stimulus) -> str:
+    dataset_name = stimulus.get("dataset_name", "").strip()
+    image_identifier = stimulus.get("image_identifier", "").strip()
+    if not dataset_name or not image_identifier:
+        raise ValueError("Stimulus must include dataset_name and image_identifier.")
+    return f"{dataset_name}::{image_identifier}"
+
+
+def resolve_stimulus_indices(
+    selected_stimuli: Iterable[Stimulus | str],
+    available_stimuli: Iterable[Stimulus],
+) -> list[int]:
+    stimulus_index = {stimulus_key(stimulus): idx for idx, stimulus in enumerate(available_stimuli)}
+    keys: list[str] = []
+    for item in selected_stimuli:
+        if isinstance(item, str):
+            key = item
+        else:
+            key = stimulus_key(item)
+        keys.append(key)
+
+    if not keys:
+        raise ValueError("Select at least one stimulus.")
+    if len(keys) != len(set(keys)):
+        raise ValueError("Stimulus selections must be unique.")
+
+    missing = [key for key in keys if key not in stimulus_index]
+    if missing:
+        missing_str = ", ".join(missing)
+        raise ValueError(f"Unknown stimuli requested: {missing_str}")
+
+    return [stimulus_index[key] for key in keys]
+
+
+@lru_cache(maxsize=1)
+def get_dummy_model_embeddings() -> dict[str, np.ndarray]:
+    rng = np.random.default_rng(2026)
+    models = list_dummy_models()
+    stimuli = list_dummy_stimuli()
+    num_samples = len(stimuli)
+    dim = 64
+
+    family_by_model = {
+        "vit_base_patch16_224": "vit",
+        "vit_large_patch16_224": "vit",
+        "resnet50": "resnet",
+        "resnet101": "resnet",
+        "convnext_base": "convnext",
+        "convnext_large": "convnext",
+        "deit_base_patch16_224": "vit",
+        "clip_vit_b32": "vit",
+        "swin_tiny_patch4_window7_224": "swin",
+    }
+
+    global_base = rng.normal(size=(num_samples, dim)).astype(np.float32)
+    family_bases = {}
+    for family in sorted(set(family_by_model.values())):
+        family_noise = rng.normal(size=(num_samples, dim)).astype(np.float32)
+        family_stimulus = rng.normal(size=(num_samples, dim)).astype(np.float32)
+        # Separate family-level structure so cross-family CKA drops more.
+        family_bases[family] = 0.15 * global_base + 0.55 * family_noise + 0.30 * family_stimulus
+
+    scales = np.linspace(0.02, 0.08, len(models))
+
+    embeddings: dict[str, np.ndarray] = {}
+    for model_name, scale in zip(models, scales):
+        family = family_by_model.get(model_name, "other")
+        base = family_bases.get(family, global_base)
+        noise = rng.normal(size=(num_samples, dim)).astype(np.float32)
+        embeddings[model_name] = (base + scale * noise).astype(np.float32)
+
+    return embeddings

src/hackathon/modal_client.py

@@ -0,0 +1,155 @@
+"""Thin client for the private Modal evaluation backend.
+
+The actual evaluation pipeline (embedding extraction, CKA scoring) lives in
+a private repository and is deployed as a Modal app.  This module calls the
+deployed functions by name — no backend code is imported here.
+
+Public configs (blue model registry) can still be controlled via HF Space env
+vars for convenience.  Secret configs (red team registry, blue heldout images)
+are loaded server-side from the Modal volume — never sent from here.
+
+The backend must be deployed first:
+    modal deploy scripts/modal_backend.py   # from the private eval-backend repo
+"""
+from __future__ import annotations
+
+import json
+import os
+from pathlib import Path
+from typing import Any, Iterable
+
+from src.hackathon.validation import (
+    BLUE_MODEL_REGISTRY_ENV,
+    MODEL_REGISTRY_ENV,
+)
+
+
+MODAL_ENABLE_ENV = "HACKATHON_MODAL_ENABLE"
+MODAL_APP_ENV = "HACKATHON_MODAL_APP"
+MODAL_BATCH_SIZE_ENV = "HACKATHON_MODAL_BATCH_SIZE"
+DEFAULT_MODAL_APP = "iclr2026-eval"
+DEFAULT_BATCH_SIZE = 64
+
+
+def _is_truthy(value: str | None) -> bool:
+    if value is None:
+        return False
+    return value.strip().lower() in {"1", "true", "yes", "y", "on"}
+
+
+def is_modal_enabled() -> bool:
+    return _is_truthy(os.environ.get(MODAL_ENABLE_ENV))
+
+
+def _get_batch_size() -> int:
+    raw = os.environ.get(MODAL_BATCH_SIZE_ENV, "").strip()
+    if raw:
+        return int(raw)
+    return DEFAULT_BATCH_SIZE
+
+
+def _get_modal_function(function_name: str) -> Any:
+    import modal
+
+    app_name = os.environ.get(MODAL_APP_ENV, "").strip() or DEFAULT_MODAL_APP
+    return modal.Function.from_name(app_name, function_name)
+
+
+def _load_json_file(path: str) -> Any:
+    """Load a JSON or JSONL file from a local path."""
+    p = Path(path)
+    if p.suffix == ".jsonl":
+        lines = p.read_text().splitlines()
+        return [json.loads(line) for line in lines if line.strip()]
+    return json.loads(p.read_text())
+
+
+def _load_blue_model_registry() -> list[dict[str, Any]] | None:
+    """Load blue model registry from env var if set, else return None.
+
+    When None is returned the backend loads its copy from the Modal volume.
+    """
+    path = os.environ.get(BLUE_MODEL_REGISTRY_ENV, "").strip()
+    if not path:
+        path = os.environ.get(MODEL_REGISTRY_ENV, "").strip()
+    if not path:
+        return None
+
+    data = _load_json_file(path)
+    if isinstance(data, dict):
+        data = data.get("models", data)
+    return data
+
+
+def score_blue_with_pairwise(
+    model_names: Iterable[str],
+    *,
+    submission_id: str | None = None,
+    submitter: str | None = None,
+) -> tuple[float, list[dict[str, Any]]]:
+    """Score a blue team submission via the deployed Modal backend.
+
+    If HACKATHON_BLUE_MODEL_REGISTRY (or HACKATHON_MODEL_REGISTRY) is set,
+    the registry is sent to the backend.  Otherwise the backend loads its
+    own copy from the Modal volume.
+
+    Blue heldout images are always loaded server-side (secret).
+    When submission_id is provided, the backend saves the result to the
+    Modal volume for crash recovery.
+    """
+    model_registry = _load_blue_model_registry()
+
+    fn = _get_modal_function("score_blue_submission")
+    result = fn.remote(
+        model_names=list(model_names),
+        model_registry=model_registry,
+        batch_size=_get_batch_size(),
+        submission_id=submission_id,
+        submitter=submitter,
+    )
+    avg_cka = float(result.get("avg_cka", 0.0))
+    return avg_cka, list(result.get("pairwise", []))
+
+
+def score_red_with_pairwise(
+    selected_stimuli: Iterable[dict[str, str] | str],
+    *,
+    submission_id: str | None = None,
+    submitter: str | None = None,
+) -> tuple[float, list[dict[str, Any]]]:
+    """Score a red team submission via the deployed Modal backend.
+
+    The red team model registry is always loaded server-side from the
+    Modal volume (secret — never sent from the public Space).
+    When submission_id is provided, the backend saves the result to the
+    Modal volume for crash recovery.
+    """
+    stimuli_list: list[dict[str, str]] = []
+    for item in selected_stimuli:
+        if isinstance(item, str):
+            parts = item.split("::", 1)
+            if len(parts) == 2:
+                stimuli_list.append({"dataset_name": parts[0], "image_identifier": parts[1]})
+            else:
+                raise ValueError(f"Invalid stimulus key format: {item}")
+        else:
+            stimuli_list.append(dict(item))
+
+    fn = _get_modal_function("score_red_submission")
+    result = fn.remote(
+        selected_stimuli=stimuli_list,
+        batch_size=_get_batch_size(),
+        submission_id=submission_id,
+        submitter=submitter,
+    )
+    score = float(result.get("score", 0.0))
+    return score, list(result.get("pairwise", []))
+
+
+def fetch_volume_submissions(team: str | None = None) -> list[dict[str, Any]]:
+    """Fetch submissions saved on the Modal volume.
+
+    Used to sync submissions after a Space restart.
+    """
+    fn = _get_modal_function("list_submissions")
+    return fn.remote(team=team)

src/hackathon/scoring.py

@@ -0,0 +1,136 @@
+from __future__ import annotations
+
+from itertools import combinations
+from typing import Iterable
+
+import numpy as np
+
+from src.cka.compute import linear_cka
+from src.hackathon.data import get_dummy_model_embeddings, list_dummy_stimuli, resolve_stimulus_indices
+from src.hackathon.modal_client import (
+    is_modal_enabled,
+    score_blue_with_pairwise as modal_score_blue_with_pairwise,
+    score_red_with_pairwise as modal_score_red_with_pairwise,
+)
+
+
+def _validate_models(model_names: Iterable[str], embeddings_by_model: dict[str, np.ndarray]) -> list[str]:
+    names = [name.strip() for name in model_names if name.strip()]
+    if len(names) < 2:
+        raise ValueError("Select at least two models.")
+    if len(names) != len(set(names)):
+        raise ValueError("Model selections must be unique.")
+
+    missing = [name for name in names if name not in embeddings_by_model]
+    if missing:
+        missing_str = ", ".join(missing)
+        raise ValueError(f"Unknown models requested: {missing_str}")
+
+    return names
+
+
+def _format_score(score: float) -> float:
+    return round(float(score), 4)
+
+
+def _pairwise_scores(
+    model_names: Iterable[str],
+    embeddings_by_model: dict[str, np.ndarray],
+) -> tuple[float, list[dict[str, float | str]]]:
+    scores = []
+    pairwise: list[dict[str, float | str]] = []
+    for model_a, model_b in combinations(model_names, 2):
+        score = linear_cka(embeddings_by_model[model_a], embeddings_by_model[model_b])
+        scores.append(score)
+        pairwise.append(
+            {
+                "Model A": model_a,
+                "Model B": model_b,
+                "CKA": _format_score(score),
+            }
+        )
+
+    if not scores:
+        return 0.0, []
+    return float(np.mean(scores)), pairwise
+
+
+def score_blue_with_pairwise(
+    model_names: Iterable[str],
+    *,
+    embeddings_by_model: dict[str, np.ndarray] | None = None,
+    submission_id: str | None = None,
+    submitter: str | None = None,
+) -> tuple[float, list[dict[str, float | str]]]:
+    if embeddings_by_model is None and is_modal_enabled():
+        avg_cka, pairwise = modal_score_blue_with_pairwise(
+            model_names, submission_id=submission_id, submitter=submitter,
+        )
+        formatted = [
+            {"Model A": item["model_a"], "Model B": item["model_b"], "CKA": _format_score(item["cka"])}
+            for item in pairwise
+        ]
+        return float(avg_cka), formatted
+
+    if embeddings_by_model is None:
+        embeddings_by_model = get_dummy_model_embeddings()
+
+    model_names = _validate_models(model_names, embeddings_by_model)
+    avg_cka, pairwise = _pairwise_scores(model_names, embeddings_by_model)
+    return float(avg_cka), pairwise
+
+
+def score_blue(
+    model_names: Iterable[str],
+    *,
+    embeddings_by_model: dict[str, np.ndarray] | None = None,
+) -> float:
+    avg_cka, _ = score_blue_with_pairwise(model_names, embeddings_by_model=embeddings_by_model)
+    return float(avg_cka)
+
+
+def score_red_with_pairwise(
+    selected_stimuli: Iterable[dict[str, str] | str],
+    *,
+    embeddings_by_model: dict[str, np.ndarray] | None = None,
+    stimuli_catalog: Iterable[dict[str, str]] | None = None,
+    submission_id: str | None = None,
+    submitter: str | None = None,
+) -> tuple[float, list[dict[str, float | str]]]:
+    if embeddings_by_model is None and is_modal_enabled():
+        score, pairwise = modal_score_red_with_pairwise(
+            selected_stimuli, submission_id=submission_id, submitter=submitter,
+        )
+        formatted = [
+            {"Model A": item["model_a"], "Model B": item["model_b"], "CKA": _format_score(item["cka"])}
+            for item in pairwise
+        ]
+        return float(score), formatted
+
+    if embeddings_by_model is None:
+        embeddings_by_model = get_dummy_model_embeddings()
+    if stimuli_catalog is None:
+        stimuli_catalog = list_dummy_stimuli()
+
+    model_names = _validate_models(embeddings_by_model.keys(), embeddings_by_model)
+    stimulus_indices = resolve_stimulus_indices(selected_stimuli, stimuli_catalog)
+    if len(stimulus_indices) < 2:
+        raise ValueError("Select at least two stimuli.")
+
+    filtered = {name: embeddings_by_model[name][stimulus_indices] for name in model_names}
+    avg_cka, pairwise = _pairwise_scores(model_names, filtered)
+    return float(1.0 - avg_cka), pairwise
+
+
+def score_red(
+    selected_stimuli: Iterable[dict[str, str] | str],
+    *,
+    embeddings_by_model: dict[str, np.ndarray] | None = None,
+    stimuli_catalog: Iterable[dict[str, str]] | None = None,
+) -> float:
+    score, _ = score_red_with_pairwise(
+        selected_stimuli,
+        embeddings_by_model=embeddings_by_model,
+        stimuli_catalog=stimuli_catalog,
+    )
+    return float(score)

src/hackathon/storage.py

@@ -0,0 +1,151 @@
+from __future__ import annotations
+
+import json
+import os
+import threading
+from typing import Any, Literal
+
+from datasets import Dataset, load_dataset
+
+TeamName = Literal["blue", "red"]
+
+DATA_DIR_ENV = "HACKATHON_DATA_DIR"
+BLUE_PATH_ENV = "HACKATHON_BLUE_DATA_PATH"
+RED_PATH_ENV = "HACKATHON_RED_DATA_PATH"
+HF_DATASET_ENV = "HACKATHON_HF_DATASET"
+HF_BLUE_DATASET_ENV = "HACKATHON_HF_BLUE_DATASET"
+HF_RED_DATASET_ENV = "HACKATHON_HF_RED_DATASET"
+HF_TOKEN_ENV = "HF_TOKEN_SUBMISSIONS"
+HF_TOKEN_FALLBACK_ENV = "HF_TOKEN"
+
+DEFAULT_DATA_DIR = os.environ.get(DATA_DIR_ENV, os.path.join(os.getcwd(), "hackathon-data"))
+DEFAULT_BLUE_PATH = os.environ.get(BLUE_PATH_ENV, os.path.join(DEFAULT_DATA_DIR, "blue_submissions.json"))
+DEFAULT_RED_PATH = os.environ.get(RED_PATH_ENV, os.path.join(DEFAULT_DATA_DIR, "red_submissions.json"))
+
+_TEAM_PATHS = {
+    "blue": DEFAULT_BLUE_PATH,
+    "red": DEFAULT_RED_PATH,
+}
+
+_LOCK = threading.Lock()
+
+
+def _resolve_hf_dataset(team: TeamName) -> str | None:
+    dataset_name = (
+        os.environ.get(HF_BLUE_DATASET_ENV, "").strip()
+        if team == "blue"
+        else os.environ.get(HF_RED_DATASET_ENV, "").strip()
+    )
+    if dataset_name:
+        return dataset_name
+
+    legacy_dataset = os.environ.get(HF_DATASET_ENV, "").strip()
+    if legacy_dataset:
+        raise ValueError(
+            "Set HACKATHON_HF_BLUE_DATASET and HACKATHON_HF_RED_DATASET for separate datasets. "
+            f"HACKATHON_HF_DATASET is no longer supported: {legacy_dataset}"
+        )
+    return None
+
+
+def _require_hf_token() -> str:
+    token = os.environ.get(HF_TOKEN_ENV) or os.environ.get(HF_TOKEN_FALLBACK_ENV)
+    if not token:
+        raise ValueError(
+            "HF_TOKEN_SUBMISSIONS (or HF_TOKEN) is required to access the private submissions dataset."
+        )
+    return token
+
+
+def _load_hf_submissions(dataset_name: str, token: str) -> list[dict[str, Any]]:
+    try:
+        dataset = load_dataset(dataset_name, split="train", token=token)
+    except Exception as exc:
+        message = str(exc).lower()
+        if any(code in message for code in ("401", "403", "permission", "unauthorized", "forbidden")):
+            raise ValueError(
+                f"HF_TOKEN_SUBMISSIONS does not have access to the private dataset: {dataset_name}"
+            ) from exc
+        if any(
+            text in message
+            for text in ("not found", "404", "doesn't exist", "no such dataset", "split")
+        ):
+            return []
+        raise
+    return dataset.to_list()
+
+
+def _save_hf_submissions(dataset_name: str, token: str, submissions: list[dict]) -> None:
+    dataset = Dataset.from_list(submissions)
+    dataset.push_to_hub(dataset_name, token=token, private=True)
+
+
+def _resolve_data_path(team: TeamName, data_path: str | None) -> str:
+    if data_path:
+        return data_path
+    if team not in _TEAM_PATHS:
+        raise ValueError(f"Unknown team: {team}")
+    return _TEAM_PATHS[team]
+
+
+def load_submissions(team: TeamName, data_path: str | None = None) -> list[dict[str, Any]]:
+    dataset_name = _resolve_hf_dataset(team)
+    if dataset_name:
+        token = _require_hf_token()
+        return _load_hf_submissions(dataset_name, token)
+
+    resolved_path = _resolve_data_path(team, data_path)
+    if not os.path.exists(resolved_path):
+        return []
+
+    with _LOCK:
+        with open(resolved_path, "r") as f:
+            data = json.load(f)
+
+    if not isinstance(data, list):
+        raise ValueError(f"Expected a list in {resolved_path}.")
+
+    return data
+
+
+def save_submissions(
+    team: TeamName,
+    submissions: list[dict[str, Any]],
+    data_path: str | None = None,
+) -> None:
+    dataset_name = _resolve_hf_dataset(team)
+    if dataset_name:
+        token = _require_hf_token()
+        _save_hf_submissions(dataset_name, token, submissions)
+        return
+
+    resolved_path = _resolve_data_path(team, data_path)
+    data_dir = os.path.dirname(resolved_path)
+    if data_dir:
+        os.makedirs(data_dir, exist_ok=True)
+
+    tmp_path = f"{resolved_path}.tmp"
+    with _LOCK:
+        with open(tmp_path, "w") as f:
+            json.dump(submissions, f, indent=2)
+        os.replace(tmp_path, resolved_path)
+
+
+def append_submission(
+    team: TeamName,
+    submission: dict[str, Any],
+    data_path: str | None = None,
+) -> list[dict[str, Any]]:
+    dataset_name = _resolve_hf_dataset(team)
+    if dataset_name:
+        token = _require_hf_token()
+        with _LOCK:
+            submissions = _load_hf_submissions(dataset_name, token)
+            submissions.append(submission)
+            _save_hf_submissions(dataset_name, token, submissions)
+        return submissions
+
+    submissions = load_submissions(team, data_path)
+    submissions.append(submission)
+    save_submissions(team, submissions, data_path)
+    return submissions

src/hackathon/validation.py

@@ -0,0 +1,232 @@
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Iterable
+
+from src.hackathon.data import list_dummy_models, list_dummy_stimuli, stimulus_key
+
+MODEL_REGISTRY_ENV = "HACKATHON_MODEL_REGISTRY"
+BLUE_MODEL_REGISTRY_ENV = "HACKATHON_BLUE_MODEL_REGISTRY"
+RED_MODEL_REGISTRY_ENV = "HACKATHON_RED_MODEL_REGISTRY"
+STIMULI_CATALOG_ENV = "HACKATHON_STIMULI_CATALOG"
+BLUE_STIMULI_CATALOG_ENV = "HACKATHON_BLUE_STIMULI_CATALOG"
+
+BLUE_TEAM_REQUIRED_MODELS = 20
+
+
+def _ensure_unique(values: list[str], label: str) -> None:
+    if len(values) != len(set(values)):
+        raise ValueError(f"{label} must be unique.")
+
+
+def _load_json(path: Path) -> object:
+    try:
+        return json.loads(path.read_text())
+    except FileNotFoundError as exc:
+        raise ValueError(f"File not found: {path}") from exc
+    except json.JSONDecodeError as exc:
+        raise ValueError(f"Invalid JSON in {path}: {exc}") from exc
+
+
+def _parse_registry_entries(path: str | None) -> list[dict | str]:
+    """Parse registry file into raw entries (strings or dicts)."""
+    if not path:
+        return []
+
+    data = _load_json(Path(path))
+    if isinstance(data, dict):
+        entries = data.get("models")
+        if entries is None:
+            raise ValueError("Model registry JSON must be a list or contain a 'models' list.")
+    elif isinstance(data, list):
+        entries = data
+    else:
+        raise ValueError("Model registry JSON must be a list or object.")
+
+    return entries
+
+
+def load_model_registry(path: str | None) -> set[str]:
+    if not path:
+        return set(list_dummy_models())
+
+    entries = _parse_registry_entries(path)
+    names: list[str] = []
+    for idx, entry in enumerate(entries, start=1):
+        if isinstance(entry, str):
+            name = entry.strip()
+        elif isinstance(entry, dict):
+            name = str(entry.get("model_name", "")).strip()
+        else:
+            raise ValueError(f"Model registry entry {idx} must be a string or object.")
+
+        if not name:
+            raise ValueError(f"Model registry entry {idx} is missing model_name.")
+        names.append(name)
+
+    _ensure_unique(names, "Model registry entries")
+    return set(names)
+
+
+def load_model_registry_specs(path: str | None) -> dict[str, dict]:
+    """Load full model specs keyed by model_name.
+
+    Returns a dict mapping model_name -> {"layer": ..., "embedding": ..., "preprocess": ...}.
+    """
+    if not path:
+        return {}
+
+    entries = _parse_registry_entries(path)
+    specs: dict[str, dict] = {}
+    for idx, entry in enumerate(entries, start=1):
+        if not isinstance(entry, dict):
+            continue
+        name = str(entry.get("model_name", "")).strip()
+        if not name:
+            continue
+        specs[name] = {
+            "layer": str(entry.get("layer", "")).strip(),
+            "embedding": str(entry.get("embedding", "flatten")).strip(),
+            "preprocess": entry.get("preprocess", {}),
+        }
+    return specs
+
+
+def load_stimuli_catalog(path: str | None) -> list[dict[str, str]]:
+    if not path:
+        return list_dummy_stimuli()
+
+    path_obj = Path(path)
+    if path_obj.suffix == ".jsonl":
+        lines = path_obj.read_text().splitlines()
+        entries = [json.loads(line) for line in lines if line.strip()]
+    else:
+        data = _load_json(path_obj)
+        if isinstance(data, dict):
+            entries = data.get("stimuli")
+            if entries is None:
+                raise ValueError("Stimuli catalog JSON must be a list or contain a 'stimuli' list.")
+        elif isinstance(data, list):
+            entries = data
+        else:
+            raise ValueError("Stimuli catalog JSON must be a list or object.")
+
+    stimuli: list[dict[str, str]] = []
+    for idx, entry in enumerate(entries, start=1):
+        if not isinstance(entry, dict):
+            raise ValueError(f"Stimulus entry {idx} must be an object.")
+        dataset_name = str(entry.get("dataset_name", "")).strip()
+        image_identifier = str(entry.get("image_identifier", "")).strip()
+        if not dataset_name or not image_identifier:
+            raise ValueError(f"Stimulus entry {idx} must include dataset_name and image_identifier.")
+        stimuli.append({"dataset_name": dataset_name, "image_identifier": image_identifier})
+
+    keys = [stimulus_key(stimulus) for stimulus in stimuli]
+    _ensure_unique(keys, "Stimuli catalog entries")
+    return stimuli
+
+
+def validate_blue_submission(
+    payload: dict,
+    *,
+    model_registry: Iterable[str] | None = None,
+    registry_specs: dict[str, dict] | None = None,
+) -> list[str]:
+    if model_registry is None:
+        model_registry = list_dummy_models()
+    registry_set = set(model_registry)
+
+    models = payload.get("models")
+    if not isinstance(models, list):
+        raise ValueError("Blue submission must include a list of models.")
+
+    names: list[str] = []
+    layer_mismatches: list[str] = []
+    missing_layers: list[str] = []
+    for idx, item in enumerate(models, start=1):
+        if isinstance(item, str):
+            name = item.strip()
+            layer_name = None
+        elif isinstance(item, dict):
+            name = str(item.get("model_name", "")).strip()
+            layer_name = str(item.get("layer_name", "")).strip() or None
+        else:
+            raise ValueError(f"Model entry {idx} must be a string or object with model_name.")
+        if not name:
+            raise ValueError(f"Model entry {idx} is missing model_name.")
+        names.append(name)
+
+        # layer_name is required when registry_specs are available
+        if registry_specs:
+            if not layer_name:
+                missing_layers.append(f"Model entry {idx} ({name}) is missing layer_name.")
+            elif name in registry_specs:
+                expected_layer = registry_specs[name].get("layer", "")
+                if layer_name != expected_layer:
+                    layer_mismatches.append(
+                        f"{name}: submitted layer_name '{layer_name}' "
+                        f"does not match registry layer '{expected_layer}'"
+                    )
+
+    _ensure_unique(names, "Model selections")
+
+    if len(names) != BLUE_TEAM_REQUIRED_MODELS:
+        raise ValueError(
+            f"Blue team submission must contain exactly {BLUE_TEAM_REQUIRED_MODELS} "
+            f"unique models, but got {len(names)}."
+        )
+
+    missing = [name for name in names if name not in registry_set]
+    if missing:
+        missing_str = ", ".join(missing)
+        raise ValueError(f"Unknown models requested: {missing_str}")
+
+    if missing_layers:
+        raise ValueError(
+            f"Missing layer_name for {len(missing_layers)} model(s):\n"
+            + "\n".join(f"  - {m}" for m in missing_layers)
+        )
+
+    if layer_mismatches:
+        raise ValueError(
+            f"Layer name mismatch for {len(layer_mismatches)} model(s):\n"
+            + "\n".join(f"  - {m}" for m in layer_mismatches)
+        )
+
+    return names
+
+
+def validate_red_submission(
+    payload: dict,
+    *,
+    stimuli_catalog: Iterable[dict[str, str]] | None = None,
+) -> list[str]:
+    if stimuli_catalog is None:
+        stimuli_catalog = list_dummy_stimuli()
+
+    images = payload.get("differentiating_images")
+    if not isinstance(images, list):
+        raise ValueError("Red submission must include differentiating_images.")
+    if len(images) < 2:
+        raise ValueError("Select at least two stimuli.")
+
+    keys: list[str] = []
+    for idx, item in enumerate(images, start=1):
+        if not isinstance(item, dict):
+            raise ValueError(f"Stimulus entry {idx} must be an object.")
+        dataset_name = str(item.get("dataset_name", "")).strip()
+        image_identifier = str(item.get("image_identifier", "")).strip()
+        if not dataset_name or not image_identifier:
+            raise ValueError(f"Stimulus entry {idx} must include dataset_name and image_identifier.")
+        keys.append(stimulus_key({"dataset_name": dataset_name, "image_identifier": image_identifier}))
+
+    _ensure_unique(keys, "Stimulus selections")
+
+    available = {stimulus_key(stimulus) for stimulus in stimuli_catalog}
+    missing = [key for key in keys if key not in available]
+    if missing:
+        missing_str = ", ".join(missing)
+        raise ValueError(f"Unknown stimuli requested: {missing_str}")
+
+    return keys

test_submissions/blue_submission.jsonl

@@ -0,0 +1,2 @@
+{"model_name": "resnet18", "layer_name": "fc"}
+{"model_name": "resnet34", "layer_name": "fc"}

test_submissions/red_submission.jsonl

@@ -0,0 +1,8 @@
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00000001.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00000002.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00000003.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00000004.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00000005.JPEG"}
+{"dataset_name": "imagenet_val", "image_identifier": "ILSVRC2012_val_00000006.JPEG"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/baking_sheet/01f362ca48a547c.png"}
+{"dataset_name": "objectnet", "image_identifier": "objectnet-1.0/images/baking_sheet/03660225e3e0440.png"}