scripts/publish_submission.py

"""Score an emailed submission and push it to the public leaderboard.

Run this on the maintainer's local machine (not on the Space). It needs:
  - `hf auth login` already done, with a user token that has write access on
    Ted412/EgoMemReason-Leaderboard (as repo owner or with a fine-grained
    write scope).
  - a local copy of the private answer key (annotations_private.json). Get
    it once with:
        hf download Ted412/EgoMemReason-Private annotations_private.json \\
            --repo-type=dataset --local-dir /path/to/somewhere

Usage:
    python scripts/publish_submission.py \\
        --submission /path/to/user_submission.json \\
        --private   /path/to/annotations_private.json \\
        --team-name "MyLab" --method-name "MyModel-8B" \\
        --model-size "8B" --uses-external no --uses-frames frames-only \\
        --method-description "Frame sampling + LoRA on top of Qwen2-VL-8B." \\
        --project-url https://... --publication-url https://arxiv.org/abs/...
"""

import argparse
import io
import json
import sys
import uuid
from datetime import datetime, timezone
from pathlib import Path

# Reuse the same scorer the Space used to use — evaluator.py sits one level up.
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
import evaluator  # noqa: E402

from huggingface_hub import HfApi  # noqa: E402


PUBLIC_DATASET = "Ted412/EgoMemReason-Leaderboard"


def build_record(sid, args, metrics):
    return {
        "submission_id": sid,
        "submitted_at_utc": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
        "hf_user_id": args.hf_user_id,
        "team_name": args.team_name,
        "method_name": args.method_name,
        "model_size": args.model_size or "",
        "uses_external_data": args.uses_external == "yes",
        "uses_video_frames": args.uses_frames,
        "method_description": args.method_description or "",
        "project_url": args.project_url or "",
        "publication_url": args.publication_url or "",
        "is_selected": True,   # maintainer-published entries are official by construction
        "metrics": metrics,
    }


def main():
    p = argparse.ArgumentParser()
    p.add_argument("--submission", required=True,
                   help="Path to the emailed submission JSON "
                        "(list of {example_id, predicted_answer}).")
    p.add_argument("--private", required=True,
                   help="Path to the local annotations_private.json.")
    p.add_argument("--team-name", required=True)
    p.add_argument("--method-name", required=True)
    p.add_argument("--model-size", default="",
                   help="e.g. 8B, 32B, API. Free text.")
    p.add_argument("--uses-external", required=True, choices=["yes", "no"])
    p.add_argument("--uses-frames", required=True,
                   choices=["frames-only", "video-only", "frames+audio",
                            "captions-only", "other"])
    p.add_argument("--method-description", default="")
    p.add_argument("--project-url", default="")
    p.add_argument("--publication-url", default="")
    p.add_argument("--hf-user-id", default="Ted412",
                   help="Recorded as the submitting user; defaults to the "
                        "maintainer since they're publishing on the "
                        "submitter's behalf.")
    p.add_argument("--dry-run", action="store_true",
                   help="Score locally and print the record, but don't push.")
    args = p.parse_args()

    try:
        metrics = evaluator.score_submission(args.submission, args.private)
    except ValueError as e:
        print(f"[publish] validation failed:\n{e}", file=sys.stderr)
        sys.exit(2)

    sid = str(uuid.uuid4())
    record = build_record(sid, args, metrics)

    print(json.dumps(record, indent=2))
    print(f"[publish] scored: Overall = {metrics['Overall']:.1f}")

    if args.dry_run:
        print("[publish] --dry-run set; not uploading.")
        return

    payload = json.dumps(record, indent=2).encode("utf-8")
    HfApi().upload_file(
        path_or_fileobj=io.BytesIO(payload),
        path_in_repo=f"submissions/{sid}.json",
        repo_id=PUBLIC_DATASET,
        repo_type="dataset",
        commit_message=(
            f"add leaderboard entry {sid[:8]}: "
            f"{args.team_name} / {args.method_name} "
            f"(Overall {metrics['Overall']:.1f})"
        ),
    )
    print(f"[publish] uploaded to "
          f"https://huggingface.co/datasets/{PUBLIC_DATASET}/blob/main/submissions/{sid}.json")


if __name__ == "__main__":
    main()