NidhiS09 commited on
Commit
d971a6a
Β·
verified Β·
1 Parent(s): 1449db6

Update main.py

Browse files
Files changed (1) hide show
  1. main.py +312 -411
main.py CHANGED
@@ -3,14 +3,11 @@ import json
3
  import uuid
4
  import time
5
  import tempfile
6
- import hashlib
7
- import hmac
8
  from datetime import datetime, timezone
9
  from typing import Any, Dict, List, Tuple, Optional
10
 
11
- import streamlit as st
12
  import pandas as pd
13
- from PIL import Image
14
  from huggingface_hub import HfApi, hf_hub_download
15
  from huggingface_hub.utils import HfHubHTTPError
16
 
@@ -18,229 +15,37 @@ from huggingface_hub.utils import HfHubHTTPError
18
  # CONFIG
19
  # =========================
20
 
21
- st.set_page_config(
22
- page_title="VizWiz Benchmark Arena",
23
- page_icon="πŸ†",
24
- layout="wide",
25
- initial_sidebar_state="expanded",
26
- )
27
-
28
- DB_REPO_ID = os.getenv("DB_REPO_ID", "NidhiS09/VizWiz-submissions-db")
29
  DB_REPO_TYPE = "dataset"
30
  SUBMISSIONS_TOKEN = os.getenv("SUBMISSIONS_TOKEN", "")
31
 
32
  DAILY_SUBMISSION_CAP = 5
33
 
34
  PHASES = [
35
- {"label": "Dev (query-dev2024)", "codename": "test-dev2024"},
36
- {"label": "Standard (query-standard2024)", "codename": "test-standard2024"},
37
- {"label": "Challenge (query-challenge2024)", "codename": "test-challenge2024"},
38
  ]
39
 
40
  CHALLENGE_TYPES = ["Object Detection", "Instance Segmentation"]
41
  LEADERBOARD_METRICS = ["bbox_mAP", "bbox_AP50", "segm_mAP", "segm_AP50"]
42
  DEFAULT_SORT_METRIC = "segm_AP50"
43
- USERS_FILE = "users.jsonl"
44
 
45
  # =========================
46
- # PASSWORD HELPERS
47
  # =========================
48
 
49
- def _hash_password(password: str) -> str:
50
- """SHA-256 hash with a salt stored in env, no bcrypt dependency needed."""
51
- salt = os.getenv("PASSWORD_SALT", "vizwiz-benchmark-salt-2024")
52
- return hashlib.sha256(f"{salt}{password}".encode()).hexdigest()
53
-
54
- def _verify_password(password: str, hashed: str) -> bool:
55
- return hmac.compare_digest(_hash_password(password), hashed)
56
-
57
- # =========================
58
- # USER STORE (users.jsonl in private dataset)
59
- # =========================
60
-
61
- @st.cache_resource
62
  def api_client() -> HfApi:
63
- return HfApi()
64
-
65
- def _load_users() -> Dict[str, Dict]:
66
- """Returns dict of username -> user record."""
67
- try:
68
- path = hf_hub_download(
69
- repo_id=DB_REPO_ID, repo_type=DB_REPO_TYPE,
70
- filename=USERS_FILE, token=SUBMISSIONS_TOKEN
71
- )
72
- users = {}
73
- with open(path, "r") as f:
74
- for line in f:
75
- line = line.strip()
76
- if line:
77
- try:
78
- u = json.loads(line)
79
- users[u["username"].lower()] = u
80
- except Exception:
81
- continue
82
- return users
83
- except HfHubHTTPError as e:
84
- if "404" in str(e):
85
- return {}
86
- raise
87
- except Exception:
88
- return {}
89
-
90
- def _save_user(user: Dict) -> None:
91
- """Append a new user to users.jsonl."""
92
- try:
93
- path = hf_hub_download(
94
- repo_id=DB_REPO_ID, repo_type=DB_REPO_TYPE,
95
- filename=USERS_FILE, token=SUBMISSIONS_TOKEN
96
- )
97
- existing = open(path, "r").read()
98
- except HfHubHTTPError as e:
99
- existing = "" if "404" in str(e) else (_ for _ in ()).throw(e)
100
-
101
- updated = (existing.rstrip("\n") + "\n" + json.dumps(user) + "\n") if existing.strip() else (json.dumps(user) + "\n")
102
-
103
- with tempfile.NamedTemporaryFile("w", suffix=".jsonl", delete=False) as f:
104
- f.write(updated)
105
- tmp = f.name
106
-
107
- api_client().upload_file(
108
- path_or_fileobj=tmp,
109
- path_in_repo=USERS_FILE,
110
- repo_id=DB_REPO_ID,
111
- repo_type=DB_REPO_TYPE,
112
- token=SUBMISSIONS_TOKEN,
113
- commit_message=f"new user: {user['username']}",
114
- )
115
- try:
116
- os.remove(tmp)
117
- except Exception:
118
- pass
119
-
120
- # =========================
121
- # AUTH STATE
122
- # =========================
123
-
124
- def get_logged_in_user() -> Optional[Dict]:
125
- return st.session_state.get("user", None)
126
-
127
- def login_user(user: Dict) -> None:
128
- st.session_state["user"] = user
129
-
130
- def logout_user() -> None:
131
- st.session_state.pop("user", None)
132
-
133
- def render_login_wall() -> Dict:
134
- """Redirect to login if not authenticated."""
135
- user = get_logged_in_user()
136
- if user:
137
- return user
138
- st.warning("Please log in to access this page.")
139
- st.session_state["redirect_after_login"] = True
140
- st.stop()
141
-
142
- # =========================
143
- # AUTH PAGES
144
- # =========================
145
-
146
- def page_login():
147
- st.header("πŸ” Login")
148
- col, _ = st.columns([1, 1])
149
- with col:
150
- username = st.text_input("Username", key="login_username").strip().lower()
151
- password = st.text_input("Password", type="password", key="login_password")
152
-
153
- if st.button("Login", type="primary", use_container_width=True):
154
- if not username or not password:
155
- st.error("Please enter username and password.")
156
- return
157
-
158
- with st.spinner("Verifying..."):
159
- users = _load_users()
160
-
161
- if username not in users:
162
- st.error("Username not found.")
163
- return
164
-
165
- user = users[username]
166
- if not _verify_password(password, user["password_hash"]):
167
- st.error("Incorrect password.")
168
- return
169
-
170
- login_user(user)
171
- st.success(f"Welcome back, {user['username']}!")
172
- st.rerun()
173
-
174
- st.markdown("---")
175
- st.caption("Don't have an account?")
176
- if st.button("Create an account β†’", use_container_width=True):
177
- st.session_state["auth_page"] = "signup"
178
- st.rerun()
179
-
180
-
181
- def page_signup():
182
- st.header("✏️ Create Account")
183
- col, _ = st.columns([1, 1])
184
- with col:
185
- username = st.text_input("Username", key="signup_username").strip()
186
- email = st.text_input("Email", key="signup_email").strip()
187
- team = st.text_input("Team / Organization", key="signup_team").strip()
188
- password = st.text_input("Password", type="password", key="signup_password")
189
- password2 = st.text_input("Confirm Password", type="password", key="signup_password2")
190
-
191
- if st.button("Create Account", type="primary", use_container_width=True):
192
- if not all([username, email, team, password, password2]):
193
- st.error("Please fill in all fields.")
194
- return
195
- if len(username) < 3:
196
- st.error("Username must be at least 3 characters.")
197
- return
198
- if password != password2:
199
- st.error("Passwords do not match.")
200
- return
201
- if len(password) < 6:
202
- st.error("Password must be at least 6 characters.")
203
- return
204
-
205
- with st.spinner("Creating account..."):
206
- users = _load_users()
207
- if username.lower() in users:
208
- st.error("Username already taken.")
209
- return
210
-
211
- new_user = {
212
- "username": username,
213
- "email": email,
214
- "team": team,
215
- "password_hash": _hash_password(password),
216
- "created_at": int(time.time()),
217
- }
218
- try:
219
- _save_user(new_user)
220
- except Exception as e:
221
- st.error(f"Failed to create account: {e}")
222
- return
223
-
224
- login_user(new_user)
225
- st.success(f"Account created! Welcome, {username}!")
226
- st.rerun()
227
-
228
- st.markdown("---")
229
- st.caption("Already have an account?")
230
- if st.button("← Back to Login", use_container_width=True):
231
- st.session_state["auth_page"] = "login"
232
- st.rerun()
233
-
234
 
235
  # =========================
236
  # SUBMISSION HELPERS
237
  # =========================
238
 
239
- def _require_token() -> None:
240
- if not SUBMISSIONS_TOKEN:
241
- st.error("Missing SUBMISSIONS_TOKEN. Add it in Space Settings β†’ Secrets.")
242
- st.stop()
243
-
244
  def _today_utc_str() -> str:
245
  return datetime.now(timezone.utc).strftime("%Y-%m-%d")
246
 
@@ -317,7 +122,8 @@ def _upload_json(data: Any, path_in_repo: str, commit_message: str = "") -> None
317
 
318
  def _create_submission_record(*, pred, team, model_name, phase_codename,
319
  challenge_type, original_filename, username, email) -> str:
320
- _require_token()
 
321
  submission_id = str(uuid.uuid4())
322
  ts = int(time.time())
323
  meta = {
@@ -339,7 +145,9 @@ def _create_submission_record(*, pred, team, model_name, phase_codename,
339
  return submission_id
340
 
341
  def _load_leaderboard_df() -> pd.DataFrame:
342
- _require_token()
 
 
343
  try:
344
  path = hf_hub_download(
345
  repo_id=DB_REPO_ID, repo_type=DB_REPO_TYPE,
@@ -347,7 +155,7 @@ def _load_leaderboard_df() -> pd.DataFrame:
347
  )
348
  except HfHubHTTPError as e:
349
  if "404" in str(e):
350
- return pd.DataFrame(columns=["team", "model", "phase_codename", *LEADERBOARD_METRICS, "timestamp"])
351
  raise
352
 
353
  rows = []
@@ -362,7 +170,7 @@ def _load_leaderboard_df() -> pd.DataFrame:
362
  continue
363
 
364
  if not rows:
365
- return pd.DataFrame(columns=["team", "model", "phase_codename", *LEADERBOARD_METRICS, "timestamp"])
366
 
367
  df = pd.DataFrame(rows)
368
  for col in ["team", "model", "phase_codename", "timestamp", *LEADERBOARD_METRICS]:
@@ -373,7 +181,8 @@ def _load_leaderboard_df() -> pd.DataFrame:
373
  return df
374
 
375
  def _load_user_submissions(username: str) -> List[Dict]:
376
- _require_token()
 
377
  try:
378
  files = api_client().list_repo_files(
379
  repo_id=DB_REPO_ID, repo_type=DB_REPO_TYPE, token=SUBMISSIONS_TOKEN
@@ -403,6 +212,7 @@ def _load_user_submissions(username: str) -> List[Dict]:
403
  except Exception:
404
  status = {"state": "unknown"}
405
  metrics = status.get("metrics", {}) if status.get("state") == "done" else {}
 
406
  results.append({
407
  "submission_id": sid,
408
  "team": meta.get("team", ""),
@@ -411,6 +221,7 @@ def _load_user_submissions(username: str) -> List[Dict]:
411
  "challenge_type": meta.get("challenge_type", ""),
412
  "timestamp": meta.get("timestamp", 0),
413
  "state": status.get("state", "unknown"),
 
414
  **metrics,
415
  })
416
  except Exception:
@@ -420,247 +231,337 @@ def _load_user_submissions(username: str) -> List[Dict]:
420
  return results
421
 
422
  # =========================
423
- # UI PAGES
424
  # =========================
425
 
426
- def render_overview():
427
- with st.expander("ℹ️ Overview of the VizWiz Benchmark Arena"):
428
- st.markdown("""
429
- **VizWiz Object Localization Challenge** β€” Submit your model predictions and get scored automatically.
430
- - πŸ“€ **Submit**: Upload your prediction JSON for Dev, Standard, or Challenge phases
431
- - πŸ† **Leaderboard**: View public rankings for the Standard phase
432
- - πŸ“‹ **My Submissions**: Track all your past submissions and scores
433
- """)
434
- try:
435
- overview_image = Image.open("src/overview_image.png").resize((600, 600))
436
- st.image(overview_image, caption="Example of an object localization task")
437
- except Exception:
438
- pass
439
-
440
- def render_eval_details():
441
- with st.expander("πŸ“ How is the Score Calculated?"):
442
- st.markdown("""
443
- Your submission is evaluated automatically against hidden ground-truth annotations.
444
- - **bbox_mAP** β€” bounding box mean average precision
445
- - **bbox_AP50** β€” bounding box AP at IoU=0.50
446
- - **segm_mAP** β€” segmentation mean average precision
447
- - **segm_AP50** β€” segmentation AP at IoU=0.50 *(default ranking)*
448
- """)
449
-
450
- def page_leaderboard():
451
- st.header("πŸ† Leaderboard β€” Standard Phase")
452
- st.caption(f"Ranked by **{DEFAULT_SORT_METRIC}** (descending). Showing Standard phase submissions only.")
453
-
454
- with st.spinner("Loading leaderboard..."):
455
- try:
456
- df = _load_leaderboard_df()
457
- except Exception as e:
458
- st.error(f"Could not load leaderboard: {e}")
459
- return
460
 
461
  if not df.empty and "phase_codename" in df.columns:
462
  df = df[df["phase_codename"] == "test-standard2024"]
463
 
464
  if df.empty:
465
- st.info("No scored Standard phase submissions yet. Be the first!")
466
- return
467
 
468
  df_display = df.copy()
469
  df_display.insert(0, "Rank", range(1, len(df_display) + 1))
470
  if "timestamp" in df_display.columns:
471
- df_display["timestamp"] = pd.to_datetime(df_display["timestamp"], unit="s", errors="coerce")
 
 
 
472
  for col in ["username", "email", "phase_codename", "submission_id"]:
473
  if col in df_display.columns:
474
  df_display.drop(columns=[col], inplace=True)
475
 
476
- st.dataframe(
477
- df_display,
478
- column_config={
479
- "Rank": st.column_config.Column("Rank", width="small"),
480
- "team": "Team", "model": "Model",
481
- "bbox_mAP": st.column_config.NumberColumn("bbox_mAP", format="%.4f"),
482
- "bbox_AP50": st.column_config.NumberColumn("bbox_AP50", format="%.4f"),
483
- "segm_mAP": st.column_config.NumberColumn("segm_mAP", format="%.4f"),
484
- "segm_AP50": st.column_config.NumberColumn("segm_AP50", format="%.4f"),
485
- "timestamp": st.column_config.DatetimeColumn("Scored at", format="D MMM YYYY, h:mm a"),
486
- },
487
- use_container_width=True, hide_index=True,
488
- )
489
 
490
- def page_submit():
491
- user = render_login_wall()
492
- username = user["username"]
493
-
494
- st.header("πŸš€ Submit Your Predictions")
495
-
496
- col_info, col_cap = st.columns([3, 1])
497
- with col_info:
498
- st.markdown(f"πŸ‘€ Logged in as **{username}** ({user.get('team', '')})")
499
- with col_cap:
500
- with st.spinner("Checking daily limit..."):
501
- subs_today = _count_submissions_today(username)
502
- remaining = DAILY_SUBMISSION_CAP - subs_today
503
- color = "green" if remaining > 1 else ("orange" if remaining == 1 else "red")
504
- st.markdown(
505
- f'<div style="text-align:center;padding:8px;border-radius:8px;border:1px solid {color}">'
506
- f'<b style="color:{color}">{remaining}/{DAILY_SUBMISSION_CAP}</b><br>'
507
- f'<small>submissions left today</small></div>',
508
- unsafe_allow_html=True,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
509
  )
 
 
510
 
511
- st.markdown("---")
512
- if remaining <= 0:
513
- st.error(f"β›” You've reached your daily limit of {DAILY_SUBMISSION_CAP} submissions. Come back tomorrow!")
514
- return
515
-
516
- col1, col2 = st.columns([2, 1])
517
- with col2:
518
- st.subheader("Submission Info")
519
- team = st.text_input("Team / Display Name", value=st.session_state.get("team", user.get("team", "")))
520
- model_name = st.text_input("Model Name", value=st.session_state.get("model_name", ""))
521
- phase_label = st.selectbox("Phase", [p["label"] for p in PHASES])
522
- phase_codename = next(p["codename"] for p in PHASES if p["label"] == phase_label)
523
- challenge_type = st.radio("Challenge type", CHALLENGE_TYPES, horizontal=False)
524
- st.session_state["team"] = team
525
- st.session_state["model_name"] = model_name
526
-
527
- with col1:
528
- st.subheader("Upload Submission File")
529
- uploaded_file = st.file_uploader("Choose a JSON file", type=["json"])
530
- if uploaded_file is None:
531
- st.info("Upload a JSON file containing a list of annotations.")
532
- return
533
- try:
534
- raw = uploaded_file.getvalue().decode("utf-8")
535
- pred_obj = json.loads(raw)
536
- except Exception:
537
- st.error("Could not parse JSON.")
538
- return
539
- ok, msg = _validate_submission_json(pred_obj)
540
- if not ok:
541
- st.error(f"Invalid submission format: {msg}")
542
- return
543
- st.success("Submission file looks valid βœ…")
544
-
545
- if st.button("Submit (Queue for Evaluation)", type="primary"):
546
- if not team.strip():
547
- st.error("Please enter a Team / Display Name.")
548
- return
549
- if not model_name.strip():
550
- st.error("Please enter a Model Name.")
551
- return
552
- if _count_submissions_today(username) >= DAILY_SUBMISSION_CAP:
553
- st.error("β›” Daily submission limit reached.")
554
- return
555
- with st.spinner("Uploading submission..."):
556
- try:
557
- submission_id = _create_submission_record(
558
- pred=pred_obj, team=team, model_name=model_name,
559
- phase_codename=phase_codename, challenge_type=challenge_type,
560
- original_filename=uploaded_file.name,
561
- username=username, email=user.get("email", ""),
562
- )
563
- except Exception as e:
564
- st.error(f"Upload failed: {e}")
565
- return
566
- st.balloons()
567
- st.success("Submission queued successfully!")
568
- st.code(f"Submission ID: {submission_id}")
569
 
570
- def page_my_submissions():
571
- user = render_login_wall()
572
- username = user["username"]
573
 
574
- st.header("πŸ“‹ My Submissions")
575
- st.caption(f"Showing all submissions for **{username}**")
 
576
 
577
- with st.spinner("Loading your submissions..."):
578
- submissions = _load_user_submissions(username)
579
 
580
  if not submissions:
581
- st.info("No submissions yet. Head to **Submit Model** to get started!")
582
- return
583
 
584
- phases_present = sorted(set(s["phase"] for s in submissions))
585
- selected_phase = st.selectbox("Filter by phase", ["All"] + phases_present)
586
- filtered = submissions if selected_phase == "All" else [s for s in submissions if s["phase"] == selected_phase]
587
 
588
- state_colors = {"queued": "🟑", "running": "πŸ”΅", "done": "🟒", "failed": "πŸ”΄", "unknown": "βšͺ"}
589
- df = pd.DataFrame(filtered)
590
  if "timestamp" in df.columns:
591
- df["submitted_at"] = pd.to_datetime(df["timestamp"], unit="s", errors="coerce")
 
 
592
  if "state" in df.columns:
593
- df["status"] = df["state"].apply(lambda s: f"{state_colors.get(s, 'βšͺ')} {s.capitalize()}")
594
 
595
- display_cols = ["submitted_at", "status", "team", "model", "phase", "challenge_type"]
596
  metric_cols = [m for m in LEADERBOARD_METRICS if m in df.columns]
597
  display_cols += metric_cols
598
  df_display = df[[c for c in display_cols if c in df.columns]].copy()
599
-
600
- col_config = {
601
- "submitted_at": st.column_config.DatetimeColumn("Submitted At", format="D MMM YYYY, h:mm a"),
602
- "status": "Status", "team": "Team", "model": "Model",
603
- "phase": "Phase", "challenge_type": "Challenge Type",
604
- }
605
  for m in metric_cols:
606
- col_config[m] = st.column_config.NumberColumn(m, format="%.4f")
 
607
 
608
- st.dataframe(df_display, column_config=col_config, use_container_width=True, hide_index=True)
609
-
610
- st.markdown("---")
611
  total = len(submissions)
612
- done = sum(1 for s in submissions if s["state"] == "done")
613
  today_count = sum(
614
  1 for s in submissions
615
  if datetime.fromtimestamp(s["timestamp"], tz=timezone.utc).strftime("%Y-%m-%d") == _today_utc_str()
616
  )
617
- c1, c2, c3 = st.columns(3)
618
- c1.metric("Total Submissions", total)
619
- c2.metric("Scored", done)
620
- c3.metric(f"Today ({DAILY_SUBMISSION_CAP} max)", f"{today_count}/{DAILY_SUBMISSION_CAP}")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
621
 
622
  # =========================
623
- # MAIN
624
  # =========================
625
 
626
- def main():
627
- st.sidebar.title("VizWiz Benchmark πŸ†")
628
- user = get_logged_in_user()
629
-
630
- if user:
631
- st.sidebar.markdown(f"πŸ‘€ **{user['username']}**")
632
- st.sidebar.caption(user.get("team", ""))
633
- if st.sidebar.button("Logout", use_container_width=True):
634
- logout_user()
635
- st.rerun()
636
- st.sidebar.markdown("---")
637
- menu = ["Leaderboard", "Submit Model", "My Submissions"]
638
- else:
639
- menu = ["Leaderboard", "Login", "Sign Up"]
640
-
641
- choice = st.sidebar.radio("Navigation", menu)
642
- st.sidebar.markdown("---")
643
- st.sidebar.caption("Submissions are queued to a private DB repo and evaluated automatically.")
644
-
645
- render_overview()
646
- render_eval_details()
647
- st.markdown("---")
648
-
649
- if choice == "Leaderboard":
650
- page_leaderboard()
651
- elif choice == "Login":
652
- if "auth_page" not in st.session_state:
653
- st.session_state["auth_page"] = "login"
654
- if st.session_state["auth_page"] == "signup":
655
- page_signup()
656
- else:
657
- page_login()
658
- elif choice == "Sign Up":
659
- page_signup()
660
- elif choice == "Submit Model":
661
- page_submit()
662
- elif choice == "My Submissions":
663
- page_my_submissions()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
664
 
665
  if __name__ == "__main__":
666
- main()
 
3
  import uuid
4
  import time
5
  import tempfile
 
 
6
  from datetime import datetime, timezone
7
  from typing import Any, Dict, List, Tuple, Optional
8
 
9
+ import gradio as gr
10
  import pandas as pd
 
11
  from huggingface_hub import HfApi, hf_hub_download
12
  from huggingface_hub.utils import HfHubHTTPError
13
 
 
15
  # CONFIG
16
  # =========================
17
 
18
+ DB_REPO_ID = os.getenv("DB_REPO_ID", "VizWiz-Challenges/submissions-db")
 
 
 
 
 
 
 
19
  DB_REPO_TYPE = "dataset"
20
  SUBMISSIONS_TOKEN = os.getenv("SUBMISSIONS_TOKEN", "")
21
 
22
  DAILY_SUBMISSION_CAP = 5
23
 
24
  PHASES = [
25
+ {"label": "Dev (test-dev2024)", "codename": "test-dev2024"},
26
+ {"label": "Standard (test-standard2024)", "codename": "test-standard2024"},
27
+ {"label": "Challenge (test-challenge2024)", "codename": "test-challenge2024"},
28
  ]
29
 
30
  CHALLENGE_TYPES = ["Object Detection", "Instance Segmentation"]
31
  LEADERBOARD_METRICS = ["bbox_mAP", "bbox_AP50", "segm_mAP", "segm_AP50"]
32
  DEFAULT_SORT_METRIC = "segm_AP50"
 
33
 
34
  # =========================
35
+ # HF API
36
  # =========================
37
 
38
+ _api = None
 
 
 
 
 
 
 
 
 
 
 
 
39
  def api_client() -> HfApi:
40
+ global _api
41
+ if _api is None:
42
+ _api = HfApi()
43
+ return _api
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
44
 
45
  # =========================
46
  # SUBMISSION HELPERS
47
  # =========================
48
 
 
 
 
 
 
49
  def _today_utc_str() -> str:
50
  return datetime.now(timezone.utc).strftime("%Y-%m-%d")
51
 
 
122
 
123
  def _create_submission_record(*, pred, team, model_name, phase_codename,
124
  challenge_type, original_filename, username, email) -> str:
125
+ if not SUBMISSIONS_TOKEN:
126
+ raise ValueError("Missing SUBMISSIONS_TOKEN.")
127
  submission_id = str(uuid.uuid4())
128
  ts = int(time.time())
129
  meta = {
 
145
  return submission_id
146
 
147
  def _load_leaderboard_df() -> pd.DataFrame:
148
+ empty = pd.DataFrame(columns=["team", "model", "phase_codename", *LEADERBOARD_METRICS, "timestamp"])
149
+ if not SUBMISSIONS_TOKEN:
150
+ return empty
151
  try:
152
  path = hf_hub_download(
153
  repo_id=DB_REPO_ID, repo_type=DB_REPO_TYPE,
 
155
  )
156
  except HfHubHTTPError as e:
157
  if "404" in str(e):
158
+ return empty
159
  raise
160
 
161
  rows = []
 
170
  continue
171
 
172
  if not rows:
173
+ return empty
174
 
175
  df = pd.DataFrame(rows)
176
  for col in ["team", "model", "phase_codename", "timestamp", *LEADERBOARD_METRICS]:
 
181
  return df
182
 
183
  def _load_user_submissions(username: str) -> List[Dict]:
184
+ if not SUBMISSIONS_TOKEN:
185
+ return []
186
  try:
187
  files = api_client().list_repo_files(
188
  repo_id=DB_REPO_ID, repo_type=DB_REPO_TYPE, token=SUBMISSIONS_TOKEN
 
212
  except Exception:
213
  status = {"state": "unknown"}
214
  metrics = status.get("metrics", {}) if status.get("state") == "done" else {}
215
+ error = status.get("error", "") if status.get("state") == "failed" else ""
216
  results.append({
217
  "submission_id": sid,
218
  "team": meta.get("team", ""),
 
221
  "challenge_type": meta.get("challenge_type", ""),
222
  "timestamp": meta.get("timestamp", 0),
223
  "state": status.get("state", "unknown"),
224
+ "error": error[:120] if error else "",
225
  **metrics,
226
  })
227
  except Exception:
 
231
  return results
232
 
233
  # =========================
234
+ # GRADIO HANDLER FUNCTIONS
235
  # =========================
236
 
237
+ def load_leaderboard():
238
+ try:
239
+ df = _load_leaderboard_df()
240
+ except Exception as e:
241
+ return pd.DataFrame(), f"❌ Could not load leaderboard: {e}"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
242
 
243
  if not df.empty and "phase_codename" in df.columns:
244
  df = df[df["phase_codename"] == "test-standard2024"]
245
 
246
  if df.empty:
247
+ return pd.DataFrame(), "ℹ️ No scored Standard phase submissions yet. Be the first!"
 
248
 
249
  df_display = df.copy()
250
  df_display.insert(0, "Rank", range(1, len(df_display) + 1))
251
  if "timestamp" in df_display.columns:
252
+ df_display["Scored At"] = pd.to_datetime(
253
+ df_display["timestamp"], unit="s", errors="coerce"
254
+ ).dt.strftime("%d %b %Y, %I:%M %p")
255
+ df_display.drop(columns=["timestamp"], inplace=True)
256
  for col in ["username", "email", "phase_codename", "submission_id"]:
257
  if col in df_display.columns:
258
  df_display.drop(columns=[col], inplace=True)
259
 
260
+ return df_display, ""
261
+
262
+
263
+ def handle_submit(file, team, model_name, phase_label, challenge_type, profile: gr.OAuthProfile | None):
264
+ if profile is None:
265
+ return "❌ You must be logged in with your HuggingFace account to submit.", ""
266
+
267
+ username = profile.username
268
+ email = getattr(profile, "email", "") or ""
 
 
 
 
269
 
270
+ if not SUBMISSIONS_TOKEN:
271
+ return "❌ Missing SUBMISSIONS_TOKEN. Add it in Space Settings β†’ Secrets.", ""
272
+
273
+ if file is None:
274
+ return "❌ Please upload a JSON file.", ""
275
+
276
+ if not team.strip():
277
+ return "❌ Please enter a Team / Display Name.", ""
278
+
279
+ if not model_name.strip():
280
+ return "❌ Please enter a Model Name.", ""
281
+
282
+ # Daily cap check
283
+ subs_today = _count_submissions_today(username)
284
+ if subs_today >= DAILY_SUBMISSION_CAP:
285
+ return f"β›” You've reached your daily limit of {DAILY_SUBMISSION_CAP} submissions. Come back tomorrow!", ""
286
+
287
+ # Parse JSON
288
+ try:
289
+ with open(file, "r", encoding="utf-8") as f:
290
+ pred_obj = json.load(f)
291
+ except Exception:
292
+ return "❌ Could not parse JSON file.", ""
293
+
294
+ # Validate
295
+ ok, msg = _validate_submission_json(pred_obj)
296
+ if not ok:
297
+ return f"❌ Invalid submission format: {msg}", ""
298
+
299
+ # Resolve phase codename
300
+ phase_codename = next((p["codename"] for p in PHASES if p["label"] == phase_label), phase_label)
301
+ original_filename = os.path.basename(file)
302
+
303
+ # Upload
304
+ try:
305
+ submission_id = _create_submission_record(
306
+ pred=pred_obj,
307
+ team=team,
308
+ model_name=model_name,
309
+ phase_codename=phase_codename,
310
+ challenge_type=challenge_type,
311
+ original_filename=original_filename,
312
+ username=username,
313
+ email=email,
314
  )
315
+ except Exception as e:
316
+ return f"❌ Upload failed: {e}", ""
317
 
318
+ remaining = DAILY_SUBMISSION_CAP - subs_today - 1
319
+ return (
320
+ f"βœ… Submission queued successfully! You have {remaining}/{DAILY_SUBMISSION_CAP} submissions remaining today.",
321
+ submission_id,
322
+ )
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
323
 
 
 
 
324
 
325
+ def load_my_submissions(phase_filter: str, profile: gr.OAuthProfile | None):
326
+ if profile is None:
327
+ return pd.DataFrame(), "❌ Please log in to view your submissions.", ""
328
 
329
+ username = profile.username
330
+ submissions = _load_user_submissions(username)
331
 
332
  if not submissions:
333
+ return pd.DataFrame(), "ℹ️ No submissions yet. Head to Submit to get started!", ""
 
334
 
335
+ if phase_filter and phase_filter != "All":
336
+ submissions = [s for s in submissions if s["phase"] == phase_filter]
 
337
 
338
+ state_icons = {"queued": "🟑", "running": "πŸ”΅", "done": "🟒", "failed": "πŸ”΄", "unknown": "βšͺ"}
339
+ df = pd.DataFrame(submissions)
340
  if "timestamp" in df.columns:
341
+ df["Submitted At"] = pd.to_datetime(
342
+ df["timestamp"], unit="s", errors="coerce"
343
+ ).dt.strftime("%d %b %Y, %I:%M %p")
344
  if "state" in df.columns:
345
+ df["Status"] = df["state"].apply(lambda s: f"{state_icons.get(s, 'βšͺ')} {s.capitalize()}")
346
 
347
+ display_cols = ["Submitted At", "Status", "team", "model", "phase", "challenge_type", "error"]
348
  metric_cols = [m for m in LEADERBOARD_METRICS if m in df.columns]
349
  display_cols += metric_cols
350
  df_display = df[[c for c in display_cols if c in df.columns]].copy()
351
+ df_display.rename(columns={
352
+ "team": "Team", "model": "Model", "phase": "Phase",
353
+ "challenge_type": "Challenge Type", "error": "Error",
354
+ }, inplace=True)
 
 
355
  for m in metric_cols:
356
+ if m in df_display.columns:
357
+ df_display[m] = df_display[m].apply(lambda x: f"{x:.4f}" if pd.notna(x) else "")
358
 
359
+ # Summary stats
 
 
360
  total = len(submissions)
361
+ done = sum(1 for s in submissions if s["state"] == "done")
362
  today_count = sum(
363
  1 for s in submissions
364
  if datetime.fromtimestamp(s["timestamp"], tz=timezone.utc).strftime("%Y-%m-%d") == _today_utc_str()
365
  )
366
+ stats = (
367
+ f"**Total:** {total} &nbsp;|&nbsp; "
368
+ f"**Scored:** {done} &nbsp;|&nbsp; "
369
+ f"**Today:** {today_count}/{DAILY_SUBMISSION_CAP}"
370
+ )
371
+
372
+ return df_display, "", stats
373
+
374
+
375
+ def get_daily_cap_info(profile: gr.OAuthProfile | None):
376
+ if profile is None:
377
+ return ""
378
+ subs_today = _count_submissions_today(profile.username)
379
+ remaining = DAILY_SUBMISSION_CAP - subs_today
380
+ return f"**{remaining}/{DAILY_SUBMISSION_CAP}** submissions remaining today"
381
+
382
+
383
+ def get_user_greeting(profile: gr.OAuthProfile | None):
384
+ if profile is None:
385
+ return "πŸ‘‹ Log in with your HuggingFace account to submit predictions."
386
+ return f"πŸ‘€ Logged in as **{profile.username}**"
387
+
388
 
389
  # =========================
390
+ # BUILD UI
391
  # =========================
392
 
393
+ OVERVIEW_MD = """
394
+ ## VizWiz Object Localization Challenge
395
+
396
+ Submit your model predictions and get scored automatically against hidden ground-truth annotations.
397
+
398
+ - πŸ“€ **Submit** β€” Upload your prediction JSON for Dev, Standard, or Challenge phases
399
+ - πŸ† **Leaderboard** β€” View public rankings for the Standard phase
400
+ - πŸ“‹ **My Submissions** β€” Track all your past submissions and scores
401
+
402
+ > Log in with your HuggingFace account to submit predictions.
403
+ """
404
+
405
+ EVAL_DETAILS_MD = """
406
+ ### How is the Score Calculated?
407
+
408
+ Your submission is evaluated automatically against hidden ground-truth annotations using **pycocotools**.
409
+
410
+ | Metric | Description |
411
+ |--------|-------------|
412
+ | `bbox_mAP` | Bounding box mean average precision |
413
+ | `bbox_AP50` | Bounding box AP at IoU = 0.50 |
414
+ | `segm_mAP` | Segmentation mean average precision |
415
+ | `segm_AP50` | Segmentation AP at IoU = 0.50 *(default ranking metric)* |
416
+ """
417
+
418
+ FORMAT_MD = """
419
+ ### Submission Format
420
+
421
+ Your JSON file must be a **list of annotation objects**, each containing:
422
+
423
+ ```json
424
+ [
425
+ {
426
+ "image_id": 123,
427
+ "category_id": 101,
428
+ "score": 0.95,
429
+ "area": 1024.0,
430
+ "bbox": [x, y, width, height],
431
+ "segmentation": [[x1, y1, x2, y2, ...]]
432
+ },
433
+ ...
434
+ ]
435
+ ```
436
+ """
437
+
438
+ with gr.Blocks(title="VizWiz Benchmark Arena", theme=gr.themes.Soft()) as demo:
439
+
440
+ gr.Markdown("# πŸ† VizWiz Benchmark Arena")
441
+
442
+ with gr.Row():
443
+ with gr.Column(scale=4):
444
+ gr.Markdown(OVERVIEW_MD)
445
+ with gr.Column(scale=1):
446
+ login_btn = gr.LoginButton(size="lg")
447
+ user_greeting = gr.Markdown("πŸ‘‹ Log in with HuggingFace to submit.")
448
+
449
+ gr.Markdown("---")
450
+
451
+ with gr.Tabs():
452
+
453
+ # ── LEADERBOARD TAB ──────────────────────────────────────────────
454
+ with gr.TabItem("πŸ† Leaderboard"):
455
+ gr.Markdown("### Standard Phase Rankings")
456
+ gr.Markdown(f"Ranked by **{DEFAULT_SORT_METRIC}** (descending). Showing Standard phase submissions only.")
457
+
458
+ with gr.Accordion("πŸ“ How is the Score Calculated?", open=False):
459
+ gr.Markdown(EVAL_DETAILS_MD)
460
+
461
+ lb_msg = gr.Markdown("")
462
+ lb_table = gr.Dataframe(interactive=False, wrap=True)
463
+ refresh_lb_btn = gr.Button("πŸ”„ Refresh Leaderboard", variant="secondary", size="sm")
464
+
465
+ def refresh_leaderboard():
466
+ df, msg = load_leaderboard()
467
+ return df, msg
468
+
469
+ refresh_lb_btn.click(refresh_leaderboard, outputs=[lb_table, lb_msg])
470
+ demo.load(refresh_leaderboard, outputs=[lb_table, lb_msg])
471
+
472
+ # ── SUBMIT TAB ───────────────────────────────────────────────────
473
+ with gr.TabItem("πŸš€ Submit Model"):
474
+ with gr.Row():
475
+ submit_greeting = gr.Markdown("πŸ‘‹ Log in with HuggingFace to submit.")
476
+ cap_info = gr.Markdown("")
477
+
478
+ gr.Markdown("---")
479
+
480
+ with gr.Row():
481
+ with gr.Column(scale=3):
482
+ gr.Markdown("#### Upload Submission File")
483
+ file_input = gr.File(label="Choose a JSON file", file_types=[".json"])
484
+ with gr.Accordion("πŸ“„ Submission Format", open=False):
485
+ gr.Markdown(FORMAT_MD)
486
+
487
+ with gr.Column(scale=2):
488
+ gr.Markdown("#### Submission Info")
489
+ team_input = gr.Textbox(label="Team / Display Name", placeholder="e.g. My Awesome Team")
490
+ model_input = gr.Textbox(label="Model Name", placeholder="e.g. ResNet50-FPN")
491
+ phase_input = gr.Dropdown(
492
+ label="Phase",
493
+ choices=[p["label"] for p in PHASES],
494
+ value=PHASES[0]["label"],
495
+ )
496
+ challenge_input = gr.Radio(
497
+ label="Challenge Type",
498
+ choices=CHALLENGE_TYPES,
499
+ value=CHALLENGE_TYPES[0],
500
+ )
501
+
502
+ submit_btn = gr.Button("Submit (Queue for Evaluation)", variant="primary", size="lg")
503
+ submit_status = gr.Markdown("")
504
+ submission_id_box = gr.Code(label="Submission ID", language=None, visible=False)
505
+
506
+ def do_submit(file, team, model_name, phase_label, challenge_type, profile: gr.OAuthProfile | None):
507
+ msg, sid = handle_submit(file, team, model_name, phase_label, challenge_type, profile)
508
+ show_id = bool(sid)
509
+ return msg, gr.update(value=sid, visible=show_id)
510
+
511
+ submit_btn.click(
512
+ do_submit,
513
+ inputs=[file_input, team_input, model_input, phase_input, challenge_input],
514
+ outputs=[submit_status, submission_id_box],
515
+ )
516
+
517
+ def update_submit_ui(profile: gr.OAuthProfile | None):
518
+ return get_user_greeting(profile), get_daily_cap_info(profile)
519
+
520
+ demo.load(update_submit_ui, outputs=[submit_greeting, cap_info])
521
+ login_btn.click(update_submit_ui, outputs=[submit_greeting, cap_info])
522
+
523
+ # ── MY SUBMISSIONS TAB ───────────────────────────────────────────
524
+ with gr.TabItem("πŸ“‹ My Submissions"):
525
+ my_sub_greeting = gr.Markdown("πŸ‘‹ Log in with HuggingFace to view your submissions.")
526
+ my_sub_stats = gr.Markdown("")
527
+
528
+ with gr.Row():
529
+ phase_filter = gr.Dropdown(
530
+ label="Filter by Phase",
531
+ choices=["All"] + [p["codename"] for p in PHASES],
532
+ value="All",
533
+ scale=2,
534
+ )
535
+ refresh_my_btn = gr.Button("πŸ”„ Refresh", variant="secondary", scale=1)
536
+
537
+ my_sub_msg = gr.Markdown("")
538
+ my_sub_table = gr.Dataframe(interactive=False, wrap=True)
539
+
540
+ def refresh_my_submissions(phase_filter, profile: gr.OAuthProfile | None):
541
+ df, msg, stats = load_my_submissions(phase_filter, profile)
542
+ greeting = get_user_greeting(profile)
543
+ return df, msg, stats, greeting
544
+
545
+ refresh_my_btn.click(
546
+ refresh_my_submissions,
547
+ inputs=[phase_filter],
548
+ outputs=[my_sub_table, my_sub_msg, my_sub_stats, my_sub_greeting],
549
+ )
550
+ phase_filter.change(
551
+ refresh_my_submissions,
552
+ inputs=[phase_filter],
553
+ outputs=[my_sub_table, my_sub_msg, my_sub_stats, my_sub_greeting],
554
+ )
555
+ demo.load(
556
+ refresh_my_submissions,
557
+ inputs=[phase_filter],
558
+ outputs=[my_sub_table, my_sub_msg, my_sub_stats, my_sub_greeting],
559
+ )
560
+
561
+ # Update top greeting on load / login
562
+ demo.load(get_user_greeting, outputs=[user_greeting])
563
+ login_btn.click(get_user_greeting, outputs=[user_greeting])
564
+
565
 
566
  if __name__ == "__main__":
567
+ demo.launch()