Update src/main.py

src/main.py

@@ -1,114 +1,394 @@
+import os
+import json
+import uuid
+import time
+import tempfile
+from typing import Any, Dict, List, Tuple
+
 import streamlit as st
 import pandas as pd
-import os
-import hashlib
-from datetime import datetime
-from pathlib import Path
-from huggingface_hub import CommitScheduler
-from localization_eval import evaluate_submission
 from PIL import Image
+from huggingface_hub import HfApi, hf_hub_download
+from huggingface_hub.utils import HfHubHTTPError
 
-# --- PERSISTENCE CONFIGURATION ---
-DATA_FILENAME = "submissions.csv"
-USER_FILENAME = "users.csv"
-DATA_DIR = Path("data")
-DATA_DIR.mkdir(exist_ok=True)
-
-DATA_PATH = DATA_DIR / DATA_FILENAME
-USER_PATH = DATA_DIR / USER_FILENAME
-
-# Initialize CommitScheduler
-# This will automatically sync everything in the /data folder to your HF Dataset
-repo_id = "VizWiz-Challenges/submissions-db" # TODO: Change this
-
-scheduler = CommitScheduler(
-    repo_id=repo_id,
-    repo_type="dataset",
-    folder_path=DATA_DIR,
-    path_in_repo="data",
-    every=5,
-    token=os.getenv("SubmissionsToken")
+
+# =========================
+# CONFIG
+# =========================
+
+st.set_page_config(
+    page_title="AI Benchmark Arena",
+    page_icon="🏆",
+    layout="wide",
+    initial_sidebar_state="expanded",
 )
 
-# --- DB HELPERS (Replaced SQLite with Pandas/CSV) ---
-
-def init_db():
-    # Create files if they don't exist
-    if not USER_PATH.exists():
-        pd.DataFrame(columns=["username", "password"]).to_csv(USER_PATH, index=False)
-    if not DATA_PATH.exists():
-        pd.DataFrame(columns=["username", "bbox_mAP", "bbox_AP50", "segm_mAP", "segm_AP50", "timestamp"]).to_csv(DATA_PATH, index=False)
-
-def make_hashes(password):
-    return hashlib.sha256(str.encode(password)).hexdigest()
-
-def add_user(username, password):
-    with scheduler.lock:
-        df = pd.read_csv(USER_PATH)
-        if username in df['username'].values:
-            return False
-        new_user = pd.DataFrame([{"username": username, "password": make_hashes(password)}])
-        df = pd.concat([df, new_user], ignore_index=True)
-        df.to_csv(USER_PATH, index=False)
-        return True
-
-def login_user(username, password):
-    df = pd.read_csv(USER_PATH)
-    user_row = df[(df['username'] == username) & (df['password'] == make_hashes(password))]
-    return not user_row.empty
-
-def save_submission(username, bbox_mAP, bbox_AP50, segm_mAP, segm_AP50):
-    with scheduler.lock:
-        df = pd.read_csv(DATA_PATH)
-        new_sub = pd.DataFrame([{
-            "username": username,
-            "bbox_mAP": bbox_mAP,
-            "bbox_AP50": bbox_AP50,
-            "segm_mAP": segm_mAP,
-            "segm_AP50": segm_AP50,
-            "timestamp": datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-        }])
-        df = pd.concat([df, new_sub], ignore_index=True)
-        df.to_csv(DATA_PATH, index=False)
-
-def get_leaderboard_data():
-    if not DATA_PATH.exists():
-        return pd.DataFrame()
-    
-    df = pd.read_csv(DATA_PATH)
+#Set this to the private dataset repo that acts as the "database"
+DB_REPO_ID = os.getenv("DB_REPO_ID", "NidhiS09/VizWiz-submissions-db")
+DB_REPO_TYPE = "dataset"
+
+# This must exist as a Space Secret in the PUBLIC UI Space
+SUBMISSIONS_TOKEN = os.getenv("SUBMISSIONS_TOKEN", "")
+
+# Phase config (copied from EvalAI config intent)
+PHASES = [
+    {"label": "Dev (qeury-dev2024)", "codename": "test-dev2024"},
+    {"label": "Standard (query-standard2024)", "codename": "test-standard2024"},
+    {"label": "Challenge (query-challenge2024)", "codename": "test-challenge2024"},
+]
+
+CHALLENGE_TYPES = ["Object Detection", "Instance Segmentation"]
+
+# Leaderboard columns from your EvalAI yaml
+LEADERBOARD_METRICS = ["bbox_mAP", "bbox_AP50", "segm_mAP", "segm_AP50"]
+DEFAULT_SORT_METRIC = "segm_AP50"
+
+
+# =========================
+# HELPERS
+# =========================
+
+def _require_token() -> None:
+    if not SUBMISSIONS_TOKEN:
+        st.error(
+            "Missing SUBMISSIONS_TOKEN. Add it in Space Settings → Secrets "
+            "(token must have read/write access ONLY to the private DB dataset repo)."
+        )
+        st.stop()
+
+
+def _validate_submission_json(obj: Any) -> Tuple[bool, str]:
+    """
+    Validates the submission format from your instructions:
+    - Top-level must be a list
+    - Each item must be a dict containing:
+      image_id (int), score (number), category_id (int), area (number),
+      bbox ([x,y,w,h]), segmentation (list)
+    """
+    if not isinstance(obj, list):
+        return False, "Submission must be a JSON list of annotations."
+
+    required_keys = {"image_id", "score", "category_id", "area", "bbox", "segmentation"}
+
+    for i, ann in enumerate(obj):
+        if not isinstance(ann, dict):
+            return False, f"Annotation at index {i} must be an object/dict."
+
+        missing = required_keys - set(ann.keys())
+        if missing:
+            return False, f"Annotation at index {i} missing keys: {sorted(list(missing))}"
+
+        # Basic type checks
+        if not isinstance(ann["image_id"], int):
+            return False, f"image_id at index {i} must be an integer."
+        if not isinstance(ann["category_id"], int):
+            return False, f"category_id at index {i} must be an integer."
+
+        if not isinstance(ann["score"], (int, float)):
+            return False, f"score at index {i} must be a number."
+
+        if not isinstance(ann["area"], (int, float)):
+            return False, f"area at index {i} must be a number."
+
+        bbox = ann["bbox"]
+        if not (isinstance(bbox, list) and len(bbox) == 4 and all(isinstance(x, (int, float)) for x in bbox)):
+            return False, f"bbox at index {i} must be a list of 4 numbers: [x, y, w, h]."
+
+        segm = ann["segmentation"]
+        if not isinstance(segm, list):
+            return False, f"segmentation at index {i} must be a list."
+
+    return True, "OK"
+
+
+def _upload_json(api: HfApi, data: Dict[str, Any] | List[Any], path_in_repo: str) -> None:
+    with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as tmp:
+        json.dump(data, tmp, ensure_ascii=False)
+        tmp_path = tmp.name
+
+    try:
+        api.upload_file(
+            path_or_fileobj=tmp_path,
+            path_in_repo=path_in_repo,
+            repo_id=DB_REPO_ID,
+            repo_type=DB_REPO_TYPE,
+            token=SUBMISSIONS_TOKEN,
+            commit_message=f"Add {path_in_repo}",
+        )
+    finally:
+        try:
+            os.remove(tmp_path)
+        except OSError:
+            pass
+
+
+def _create_submission_record(
+    *,
+    pred: List[Dict[str, Any]],
+    team: str,
+    model_name: str,
+    phase_codename: str,
+    challenge_type: str,
+    original_filename: str,
+) -> str:
+    """
+    Writes pred/meta/status to the private DB dataset repo.
+    Returns submission_id.
+    """
+    _require_token()
+    api = HfApi()
+
+    submission_id = str(uuid.uuid4())
+    ts = int(time.time())
+
+    meta = {
+        "submission_id": submission_id,
+        "team": team.strip(),
+        "model": model_name.strip(),
+        "phase_codename": phase_codename,
+        "challenge_type": challenge_type,
+        "timestamp": ts,
+        "original_filename": original_filename,
+    }
+
+    status = {"state": "queued", "timestamp": ts}
+
+    base = f"submissions/{submission_id}"
+    _upload_json(api, pred, f"{base}/pred.json")
+    _upload_json(api, meta, f"{base}/meta.json")
+    _upload_json(api, status, f"{base}/status.json")
+
+    return submission_id
+
+
+def _download_leaderboard_jsonl() -> str | None:
+    """
+    Downloads leaderboard.jsonl from the DB repo.
+    Returns local path or None if missing.
+    """
+    _require_token()
+    try:
+        return hf_hub_download(
+            repo_id=DB_REPO_ID,
+            repo_type=DB_REPO_TYPE,
+            filename="leaderboard.jsonl",
+            token=SUBMISSIONS_TOKEN,
+        )
+    except HfHubHTTPError as e:
+        # Most common: 404 when file doesn't exist yet
+        if "404" in str(e):
+            return None
+        raise
+
+
+def _load_leaderboard_df() -> pd.DataFrame:
+    path = _download_leaderboard_jsonl()
+    if path is None:
+        return pd.DataFrame(columns=["team", "model", "phase_codename", *LEADERBOARD_METRICS, "timestamp"])
+
+    rows = []
+    with open(path, "r", encoding="utf-8") as f:
+        for line in f:
+            line = line.strip()
+            if not line:
+                continue
+            try:
+                rows.append(json.loads(line))
+            except json.JSONDecodeError:
+                # Skip malformed lines rather than crashing the UI
+                continue
+
+    if not rows:
+        return pd.DataFrame(columns=["team", "model", "phase_codename", *LEADERBOARD_METRICS, "timestamp"])
+
+    df = pd.DataFrame(rows)
+
+    # Ensure columns exist
+    for col in ["team", "model", "phase_codename", "timestamp", *LEADERBOARD_METRICS]:
+        if col not in df.columns:
+            df[col] = None
+
+    # Sort descending by default metric
+    if DEFAULT_SORT_METRIC in df.columns:
+        df = df.sort_values(by=DEFAULT_SORT_METRIC, ascending=False, kind="mergesort")
+
+    return df
+
+
+# =========================
+# UI
+# =========================
+
+def render_overview():
+    with st.expander("ℹ️ Overview of the AI Benchmark Arena"):
+        st.markdown(
+            """
+
+**Note:** This Hugging Face Space queues submissions for evaluation and persists results in a private database repo.
+            """
+        )
+        # Keep this optional so missing image doesn't crash the Space
+        try:
+            overview_image = Image.open("src/overview_image.png").resize((600, 600))
+            st.image(overview_image, caption="Example of an object localization task")
+        except Exception:
+            st.info("Overview image not found at src/overview_image.png (optional).")
+
+
+def render_eval_details():
+    with st.expander("📐 How is the Score Calculated?"):
+        st.markdown(
+            """
+Your submission is evaluated offline by a private evaluator against hidden ground-truth annotations.
+The leaderboard reports:
+
+- bbox_mAP
+- bbox_AP50
+- segm_mAP
+- segm_AP50 (default ranking)
+
+Raw submissions are kept private; only scores and metadata are shown.
+            """
+        )
+
+
+def page_submit():
+    st.header("🚀 Submit your Predictions")
+
+    col1, col2 = st.columns([2, 1])
+
+    with col2:
+        st.subheader("Submission Info")
+        team = st.text_input("Team / Display Name", value=st.session_state.get("team", ""))
+        model_name = st.text_input("Model Name", value=st.session_state.get("model_name", ""))
+
+        phase_label = st.selectbox("Phase", [p["label"] for p in PHASES])
+        phase_codename = next(p["codename"] for p in PHASES if p["label"] == phase_label)
+
+        challenge_type = st.radio("Challenge type", CHALLENGE_TYPES, horizontal=False)
+
+        st.session_state["team"] = team
+        st.session_state["model_name"] = model_name
+
+        st.caption("Your submission will be queued for evaluation. Scores appear on the leaderboard after processing.")
+
+    with col1:
+        st.subheader("Upload Submission File")
+        uploaded_file = st.file_uploader("Choose a JSON file", type=["json"])
+
+        if uploaded_file is None:
+            st.info("Upload a JSON file that contains a list of annotations.")
+            return
+
+        # Parse JSON
+        try:
+            raw = uploaded_file.getvalue().decode("utf-8")
+            pred_obj = json.loads(raw)
+        except Exception:
+            st.error("Could not parse JSON. Please upload a valid JSON file.")
+            return
+
+        ok, msg = _validate_submission_json(pred_obj)
+        if not ok:
+            st.error(f"Invalid submission format: {msg}")
+            return
+
+        st.success("Submission file looks valid ✅")
+
+        submit_clicked = st.button("Submit (Queue for Evaluation)", type="primary")
+
+        if submit_clicked:
+            if not team.strip():
+                st.error("Please enter Team / Display Name.")
+                return
+            if not model_name.strip():
+                st.error("Please enter Model Name.")
+                return
+
+            with st.spinner("Uploading submission to the private database repo..."):
+                try:
+                    submission_id = _create_submission_record(
+                        pred=pred_obj,
+                        team=team,
+                        model_name=model_name,
+                        phase_codename=phase_codename,
+                        challenge_type=challenge_type,
+                        original_filename=uploaded_file.name,
+                    )
+                except Exception as e:
+                    st.error(f"Upload failed: {e}")
+                    return
+
+            st.balloons()
+            st.success("Submission queued successfully!")
+            st.code(f"Submission ID: {submission_id}")
+
+            st.info("Next: the private evaluator will score your submission and update the leaderboard.")
+
+
+def page_leaderboard():
+    st.header("🏆 Leaderboard")
+    st.write(f"Ranked by **{DEFAULT_SORT_METRIC}** (descending).")
+
+    with st.spinner("Loading leaderboard from private database repo..."):
+        try:
+            df = _load_leaderboard_df()
+        except Exception as e:
+            st.error(f"Could not load leaderboard: {e}")
+            return
+
     if df.empty:
-        return df
-
-    # Logic to get the BEST score per user
-    # Sorting by segm_mAP (desc) and timestamp (asc) to get best/earliest
-    df = df.sort_values(by=['segm_mAP', 'timestamp'], ascending=[False, True])
-    df_best = df.drop_duplicates(subset='username', keep='first')
-    
-    df_best = df_best.rename(columns={'segm_mAP': 'Best_segm_mAP'})
-    return df_best
-
-# --- CONFIGURATION & SETUP ---
-st.set_page_config(page_title="AI Benchmark Arena", page_icon="🏆", layout="wide")
-
-
-def main_app():
-   
-    if st.button("Evaluate"):
-        with st.spinner('Calculating...'):
-            bbox_mAP, bbox_AP50, segm_mAP, segm_AP50 = evaluate_submission("src/biv_query.json", save_path)
-            if bbox_mAP is not None:
-                save_submission(st.session_state['username'], bbox_mAP, bbox_AP50, segm_mAP, segm_AP50)
-                st.balloons()
-                st.success("Submission Successful!")
-
-if __name__ == '__main__':
-    init_db()
-    
-    if 'logged_in' not in st.session_state:
-        st.session_state['logged_in'] = False
-        st.session_state['username'] = None
-
-    if not st.session_state['logged_in']:
-        ui_login_signup()
+        st.info("No scored submissions yet. Submit a model to get started!")
+        return
+
+    # Add Rank column
+    df_display = df.copy()
+    df_display.insert(0, "Rank", range(1, len(df_display) + 1))
+
+    # Optional: pretty timestamp
+    if "timestamp" in df_display.columns:
+        df_display["timestamp"] = pd.to_datetime(df_display["timestamp"], unit="s", errors="coerce")
+
+    st.dataframe(
+        df_display,
+        column_config={
+            "Rank": st.column_config.Column("Rank", width="small"),
+            "team": "Team",
+            "model": "Model",
+            "phase_codename": "Phase",
+            "bbox_mAP": st.column_config.NumberColumn("bbox_mAP", format="%.4f"),
+            "bbox_AP50": st.column_config.NumberColumn("bbox_AP50", format="%.4f"),
+            "segm_mAP": st.column_config.NumberColumn("segm_mAP", format="%.4f"),
+            "segm_AP50": st.column_config.NumberColumn("segm_AP50", format="%.4f"),
+            "timestamp": st.column_config.DatetimeColumn("Scored at", format="D MMM YYYY, h:mm a"),
+        },
+        use_container_width=True,
+        hide_index=True,
+    )
+
+
+def main():
+    st.sidebar.title("AI Benchmark Arena 🏆")
+
+    # Warn early if DB repo isn't configured
+    if DB_REPO_ID.startswith("NidhiS09/"):
+        st.sidebar.warning("Set DB_REPO_ID env var or hardcode your private DB dataset repo id in main.py.")
+
+    menu = ["Submit Model", "Leaderboard"]
+    choice = st.sidebar.radio("Navigation", menu)
+
+    st.sidebar.markdown("---")
+    st.sidebar.caption("This Space queues submissions to a private DB repo and reads leaderboard results from it.")
+
+    render_overview()
+    render_eval_details()
+    st.markdown("---")
+
+    if choice == "Submit Model":
+        page_submit()
     else:
-        main_app()
+        page_leaderboard()
+
+
+if __name__ == "__main__":
+    main()