"""Score an emailed submission and push it to the public leaderboard. Run this on the maintainer's local machine (not on the Space). It needs: - `hf auth login` already done, with a user token that has write access on Ted412/EgoMemReason-Leaderboard (as repo owner or with a fine-grained write scope). - a local copy of the private answer key (annotations_private.json). Get it once with: hf download Ted412/EgoMemReason-Private annotations_private.json \\ --repo-type=dataset --local-dir /path/to/somewhere Usage: python scripts/publish_submission.py \\ --submission /path/to/user_submission.json \\ --private /path/to/annotations_private.json \\ --team-name "MyLab" --method-name "MyModel-8B" \\ --model-size "8B" --uses-external no --uses-frames frames-only \\ --method-description "Frame sampling + LoRA on top of Qwen2-VL-8B." \\ --project-url https://... --publication-url https://arxiv.org/abs/... """ import argparse import io import json import sys import uuid from datetime import datetime, timezone from pathlib import Path # Reuse the same scorer the Space used to use — evaluator.py sits one level up. sys.path.insert(0, str(Path(__file__).resolve().parents[1])) import evaluator # noqa: E402 from huggingface_hub import HfApi # noqa: E402 PUBLIC_DATASET = "Ted412/EgoMemReason-Leaderboard" def build_record(sid, args, metrics): return { "submission_id": sid, "submitted_at_utc": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), "hf_user_id": args.hf_user_id, "team_name": args.team_name, "method_name": args.method_name, "model_size": args.model_size or "", "uses_external_data": args.uses_external == "yes", "uses_video_frames": args.uses_frames, "method_description": args.method_description or "", "project_url": args.project_url or "", "publication_url": args.publication_url or "", "is_selected": True, # maintainer-published entries are official by construction "metrics": metrics, } def main(): p = argparse.ArgumentParser() p.add_argument("--submission", required=True, help="Path to the emailed submission JSON " "(list of {example_id, predicted_answer}).") p.add_argument("--private", required=True, help="Path to the local annotations_private.json.") p.add_argument("--team-name", required=True) p.add_argument("--method-name", required=True) p.add_argument("--model-size", default="", help="e.g. 8B, 32B, API. Free text.") p.add_argument("--uses-external", required=True, choices=["yes", "no"]) p.add_argument("--uses-frames", required=True, choices=["frames-only", "video-only", "frames+audio", "captions-only", "other"]) p.add_argument("--method-description", default="") p.add_argument("--project-url", default="") p.add_argument("--publication-url", default="") p.add_argument("--hf-user-id", default="Ted412", help="Recorded as the submitting user; defaults to the " "maintainer since they're publishing on the " "submitter's behalf.") p.add_argument("--dry-run", action="store_true", help="Score locally and print the record, but don't push.") args = p.parse_args() try: metrics = evaluator.score_submission(args.submission, args.private) except ValueError as e: print(f"[publish] validation failed:\n{e}", file=sys.stderr) sys.exit(2) sid = str(uuid.uuid4()) record = build_record(sid, args, metrics) print(json.dumps(record, indent=2)) print(f"[publish] scored: Overall = {metrics['Overall']:.1f}") if args.dry_run: print("[publish] --dry-run set; not uploading.") return payload = json.dumps(record, indent=2).encode("utf-8") HfApi().upload_file( path_or_fileobj=io.BytesIO(payload), path_in_repo=f"submissions/{sid}.json", repo_id=PUBLIC_DATASET, repo_type="dataset", commit_message=( f"add leaderboard entry {sid[:8]}: " f"{args.team_name} / {args.method_name} " f"(Overall {metrics['Overall']:.1f})" ), ) print(f"[publish] uploaded to " f"https://huggingface.co/datasets/{PUBLIC_DATASET}/blob/main/submissions/{sid}.json") if __name__ == "__main__": main()