code/claim3_benchmark.py

"""Claim 3 proxy: the 1,017-photo end-to-end benchmark composition.

The paper's exact 1,017-image test set is NOT publicly released (the GitHub
repo is a project page only; no dataset on the Hub). We therefore verify the
*claim as stated* -- that the benchmark comprises 1,017 real-world photographs
spanning six categories (portraits, landscapes, urban scenes, food, objects,
low-light) -- and build a tiny toy proxy test set that mirrors the stated
category structure so downstream comparison (Claim 4) has a concrete,
reproducible input. We clearly flag that the scale (6 vs 1,017) is toy.
"""
import os, json
from PIL import Image


CATEGORIES = ["portrait", "landscape", "urban", "food", "object", "lowlight"]
PAPER_TOTAL = 1017


def main():
    out = {
        "claim": "Claim 3 (proxy)",
        "paper_total": PAPER_TOTAL,
        "paper_categories": CATEGORIES,
        "dataset_released": False,
        "note": "exact 1,017 set unreleased; toy proxy of 1 image/category",
    }
    # Build toy proxy test set (1 image per category = 6 images).
    present = []
    for c in CATEGORIES:
        p = f"inputs/{c}.jpg"
        if os.path.exists(p):
            im = Image.open(p).convert("RGB")
            present.append({"category": c, "size": list(im.size)})
    out["proxy_total"] = len(present)
    out["proxy_categories_present"] = present
    out["matches_category_set"] = set(c["category"] for c in present) == set(CATEGORIES)
    os.makedirs("outputs", exist_ok=True)
    with open("outputs/claim3_benchmark.json", "w") as f:
        json.dump(out, f, indent=2)
    print(json.dumps(out, indent=2))


if __name__ == "__main__":
    main()