Upload 7 files

app.py

@@ -0,0 +1,774 @@
+import datetime
+import io
+import json
+import os
+import re
+from urllib.parse import urlparse
+
+import gradio as gr
+import pandas as pd
+from huggingface_hub import HfApi, hf_hub_download
+
+
+APP_NAME = "miniapp"
+
+# 在 Space 里通过 Secrets 配置：
+# - HF_TOKEN: 具有写 dataset 权限的 token（Settings -> Variables and secrets -> Secrets）
+# - LEADERBOARD_DATASET: 形如 "your-username/miniapp-leaderboard"（repo_type=dataset）
+HF_TOKEN = os.environ.get("HF_TOKEN") or os.environ.get("TOKEN") or os.environ.get("HUGGINGFACE_TOKEN")
+LEADERBOARD_DATASET = os.environ.get("LEADERBOARD_DATASET", "").strip()
+
+# 判断是否运行在 Hugging Face Spaces
+IN_SPACES = bool(
+    os.environ.get("SPACE_ID")
+    or os.environ.get("SPACE_REPO_NAME")
+    or os.environ.get("SPACE_AUTHOR_NAME")
+    or os.environ.get("system", "") == "spaces"
+)
+
+MAX_ENTRIES = int(os.environ.get("MAX_ENTRIES", "200"))
+
+
+def _is_valid_http_url(url: str) -> bool:
+    try:
+        parsed = urlparse(url)
+        return parsed.scheme in ("http", "https") and bool(parsed.netloc)
+    except Exception:
+        return False
+
+
+def _slug(s: str, max_len: int = 60) -> str:
+    s = (s or "").strip().lower()
+    s = re.sub(r"[^a-z0-9]+", "-", s)
+    s = re.sub(r"-{2,}", "-", s).strip("-")
+    return (s[:max_len] or "model")
+
+
+def _api() -> HfApi:
+    return HfApi(token=HF_TOKEN)
+
+
+def _ensure_dataset_repo():
+    if not HF_TOKEN:
+        raise RuntimeError("未配置 HF_TOKEN（Space Secrets）。")
+    if not LEADERBOARD_DATASET:
+        raise RuntimeError("未配置 LEADERBOARD_DATASET（例如：your-username/miniapp-leaderboard）。")
+    api = _api()
+    try:
+        api.repo_info(repo_id=LEADERBOARD_DATASET, repo_type="dataset")
+    except Exception:
+        # 不存在则创建（public dataset；你也可以手动创建并设为 private）
+        api.create_repo(repo_id=LEADERBOARD_DATASET, repo_type="dataset", private=False, exist_ok=True)
+
+
+def _empty_df() -> pd.DataFrame:
+    return pd.DataFrame(columns=["submitted_at", "username", "model_name", "model_api", "notes"])
+
+
+def _load_submissions_df() -> pd.DataFrame:
+    if not HF_TOKEN or not LEADERBOARD_DATASET:
+        return _empty_df()
+
+    api = _api()
+    try:
+        files = api.list_repo_files(repo_id=LEADERBOARD_DATASET, repo_type="dataset")
+    except Exception:
+        return _empty_df()
+
+    sub_files = sorted(
+        [f for f in files if f.startswith("submissions/") and f.endswith(".json")],
+        reverse=True,
+    )[:MAX_ENTRIES]
+
+    rows = []
+    for filename in sub_files:
+        try:
+            path = hf_hub_download(
+                repo_id=LEADERBOARD_DATASET,
+                repo_type="dataset",
+                filename=filename,
+                token=HF_TOKEN,
+            )
+            with open(path, "r", encoding="utf-8") as fp:
+                rows.append(json.load(fp))
+        except Exception:
+            continue
+
+    if not rows:
+        return _empty_df()
+
+    df = pd.DataFrame(rows)
+    for col in ["submitted_at", "username", "model_name", "model_api", "notes"]:
+        if col not in df.columns:
+            df[col] = ""
+    df = df[["submitted_at", "username", "model_name", "model_api", "notes"]]
+    df = df.sort_values(by=["submitted_at"], ascending=False, kind="stable")
+    return df
+
+
+def refresh():
+    return _load_submissions_df()
+
+
+def submit(model_name: str, model_api: str, notes: str, username: str | None):
+    model_name = (model_name or "").strip()
+    model_api = (model_api or "").strip()
+    notes = (notes or "").strip()
+    username = (username or "").strip() or "anonymous"
+
+    if not model_name:
+        return "请填写 **模型名称**。", _load_submissions_df()
+    if not model_api:
+        return "请填写 **模型 API**。", _load_submissions_df()
+    if not _is_valid_http_url(model_api):
+        return "**模型 API** 需要是合法的 `http(s)://...` URL。", _load_submissions_df()
+
+    if not HF_TOKEN:
+        return "Space 未配置 **HF_TOKEN**（Secrets），无法写入排行榜。", _load_submissions_df()
+    if not LEADERBOARD_DATASET:
+        return "Space 未配置 **LEADERBOARD_DATASET**（例如：`your-username/miniapp-leaderboard`）。", _load_submissions_df()
+
+    _ensure_dataset_repo()
+    api = _api()
+
+    now = datetime.datetime.utcnow().replace(microsecond=0).isoformat() + "Z"
+    safe_model = _slug(model_name)
+    safe_user = _slug(username)
+    path_in_repo = f"submissions/{now[:10]}/{now}-{safe_user}-{safe_model}.json"
+
+    payload = {
+        "submitted_at": now,
+        "username": username,
+        "model_name": model_name,
+        "model_api": model_api,
+        "notes": notes,
+    }
+    data = (json.dumps(payload, ensure_ascii=False, indent=2) + "\n").encode("utf-8")
+    bio = io.BytesIO(data)
+
+    api.upload_file(
+        repo_id=LEADERBOARD_DATASET,
+        repo_type="dataset",
+        path_or_fileobj=bio,
+        path_in_repo=path_in_repo,
+        commit_message=f"miniapp: submit {username}/{model_name}",
+        token=HF_TOKEN,
+    )
+
+    return "已提交并写入 leaderboard。", _load_submissions_df()
+
+
+def build_demo() -> gr.Blocks:
+    with gr.Blocks(title=f"{APP_NAME} leaderboard") as demo:
+        gr.Markdown(
+            f"## {APP_NAME} leaderboard\n\n"
+            "提交你的模型信息后，会写入一个 Hugging Face **Dataset**，并在下方表格展示。\n\n"
+            f"- 当前 `LEADERBOARD_DATASET`: `{LEADERBOARD_DATASET or '(未配置)'}`\n"
+        )
+
+        with gr.Row():
+            with gr.Column(scale=2):
+                model_name = gr.Textbox(label="模型名称（必填）", placeholder="例如：my-agent-v1")
+                model_api = gr.Textbox(
+                    label="模型 API（必填）",
+                    placeholder="例如：https://api.example.com/v1/chat/completions",
+                )
+                notes = gr.Textbox(label="备注（可选）", lines=4)
+
+                # 纯前端版：不强制 OAuth；如果你想“只能登录用户提交”，后续再加 LoginButton
+                if IN_SPACES:
+                    username = gr.Textbox(
+                        label="用户名（可选）",
+                        placeholder="建议填你的 HF 用户名（也可留空）",
+                    )
+                else:
+                    username = gr.Textbox(label="用户名（本地调试用）", value="local")
+
+                submit_btn = gr.Button("提交", variant="primary")
+                status = gr.Markdown()
+
+            with gr.Column(scale=3):
+                leaderboard = gr.Dataframe(
+                    label="Leaderboard（按提交时间倒序）",
+                    value=_load_submissions_df(),
+                    interactive=False,
+                    wrap=True,
+                )
+                refresh_btn = gr.Button("刷新")
+
+        submit_btn.click(
+            submit,
+            inputs=[model_name, model_api, notes, username],
+            outputs=[status, leaderboard],
+        )
+        refresh_btn.click(refresh, inputs=[], outputs=[leaderboard])
+
+    return demo
+
+
+demo = build_demo()
+
+
+def main():
+    demo.launch()
+
+
+if __name__ == "__main__":
+    main()
+
+import datetime
+import io
+import json
+import os
+import re
+from urllib.parse import urlparse
+
+import gradio as gr
+import pandas as pd
+from huggingface_hub import HfApi, hf_hub_download
+
+
+APP_NAME = "miniapp"
+
+# 在 Space 里通过 Secrets 配置：
+# - HF_TOKEN: 具有写 dataset 权限的 token（Settings -> Variables and secrets -> Secrets）
+# - LEADERBOARD_DATASET: 形如 "your-username/miniapp-leaderboard"（repo_type=dataset）
+HF_TOKEN = os.environ.get("HF_TOKEN") or os.environ.get("TOKEN") or os.environ.get("HUGGINGFACE_TOKEN")
+LEADERBOARD_DATASET = os.environ.get("LEADERBOARD_DATASET", "").strip()
+
+# 判断是否运行在 Hugging Face Spaces
+IN_SPACES = bool(
+    os.environ.get("SPACE_ID")
+    or os.environ.get("SPACE_REPO_NAME")
+    or os.environ.get("SPACE_AUTHOR_NAME")
+    or os.environ.get("system", "") == "spaces"
+)
+
+MAX_ENTRIES = int(os.environ.get("MAX_ENTRIES", "200"))
+
+
+def _is_valid_http_url(url: str) -> bool:
+    try:
+        parsed = urlparse(url)
+        return parsed.scheme in ("http", "https") and bool(parsed.netloc)
+    except Exception:
+        return False
+
+
+def _slug(s: str, max_len: int = 60) -> str:
+    s = (s or "").strip().lower()
+    s = re.sub(r"[^a-z0-9]+", "-", s)
+    s = re.sub(r"-{2,}", "-", s).strip("-")
+    return (s[:max_len] or "model")
+
+
+def _api() -> HfApi:
+    return HfApi(token=HF_TOKEN)
+
+
+def _ensure_dataset_repo():
+    if not HF_TOKEN:
+        raise RuntimeError("未配置 HF_TOKEN（Space Secrets）。")
+    if not LEADERBOARD_DATASET:
+        raise RuntimeError("未配置 LEADERBOARD_DATASET（例如：your-username/miniapp-leaderboard）。")
+    api = _api()
+    try:
+        api.repo_info(repo_id=LEADERBOARD_DATASET, repo_type="dataset")
+    except Exception:
+        # 不存在则创建（public dataset；你也可以手动创建并设为 private）
+        api.create_repo(repo_id=LEADERBOARD_DATASET, repo_type="dataset", private=False, exist_ok=True)
+
+
+def _load_submissions_df() -> pd.DataFrame:
+    if not HF_TOKEN or not LEADERBOARD_DATASET:
+        return pd.DataFrame(columns=["submitted_at", "username", "model_name", "model_api", "notes"])
+
+    api = _api()
+    try:
+        files = api.list_repo_files(repo_id=LEADERBOARD_DATASET, repo_type="dataset")
+    except Exception:
+        return pd.DataFrame(columns=["submitted_at", "username", "model_name", "model_api", "notes"])
+
+    sub_files = sorted(
+        [f for f in files if f.startswith("submissions/") and f.endswith(".json")],
+        reverse=True,
+    )[:MAX_ENTRIES]
+
+    rows = []
+    for filename in sub_files:
+        try:
+            path = hf_hub_download(
+                repo_id=LEADERBOARD_DATASET,
+                repo_type="dataset",
+                filename=filename,
+                token=HF_TOKEN,
+            )
+            with open(path, "r", encoding="utf-8") as fp:
+                rows.append(json.load(fp))
+        except Exception:
+            continue
+
+    if not rows:
+        return pd.DataFrame(columns=["submitted_at", "username", "model_name", "model_api", "notes"])
+
+    df = pd.DataFrame(rows)
+    # 统一列顺序
+    for col in ["submitted_at", "username", "model_name", "model_api", "notes"]:
+        if col not in df.columns:
+            df[col] = ""
+    df = df[["submitted_at", "username", "model_name", "model_api", "notes"]]
+    df = df.sort_values(by=["submitted_at"], ascending=False, kind="stable")
+    return df
+
+
+def refresh():
+    return _load_submissions_df()
+
+
+def submit(model_name: str, model_api: str, notes: str, username: str | None):
+    model_name = (model_name or "").strip()
+    model_api = (model_api or "").strip()
+    notes = (notes or "").strip()
+    username = (username or "").strip() or "anonymous"
+
+    if not model_name:
+        return "请填写 **模型名称**。", _load_submissions_df()
+    if not model_api:
+        return "请填写 **模型 API**。", _load_submissions_df()
+    if not _is_valid_http_url(model_api):
+        return "**模型 API** 需要是合法的 `http(s)://...` URL。", _load_submissions_df()
+
+    if not HF_TOKEN:
+        return "Space 未配置 **HF_TOKEN**（Secrets），无法写入排行榜。", _load_submissions_df()
+    if not LEADERBOARD_DATASET:
+        return "Space 未配置 **LEADERBOARD_DATASET**（例如：`your-username/miniapp-leaderboard`）。", _load_submissions_df()
+
+    _ensure_dataset_repo()
+    api = _api()
+
+    now = datetime.datetime.utcnow().replace(microsecond=0).isoformat() + "Z"
+    safe_model = _slug(model_name)
+    safe_user = _slug(username)
+    path_in_repo = f"submissions/{now[:10]}/{now}-{safe_user}-{safe_model}.json"
+
+    payload = {
+        "submitted_at": now,
+        "username": username,
+        "model_name": model_name,
+        "model_api": model_api,
+        "notes": notes,
+    }
+    data = (json.dumps(payload, ensure_ascii=False, indent=2) + "\n").encode("utf-8")
+    bio = io.BytesIO(data)
+
+    api.upload_file(
+        repo_id=LEADERBOARD_DATASET,
+        repo_type="dataset",
+        path_or_fileobj=bio,
+        path_in_repo=path_in_repo,
+        commit_message=f"miniapp: submit {username}/{model_name}",
+        token=HF_TOKEN,
+    )
+
+    return "已提交并写入 leaderboard。", _load_submissions_df()
+
+
+with gr.Blocks(title=f"{APP_NAME} leaderboard") as demo:
+    gr.Markdown(
+        f"## {APP_NAME} leaderboard\n\n"
+        "提交你的模型信息后，会写入一个 Hugging Face **Dataset**，并在下方表格展示。\n\n"
+        f"- 当前 `LEADERBOARD_DATASET`: `{LEADERBOARD_DATASET or '(未配置)'}`\n"
+    )
+
+    with gr.Row():
+        with gr.Column(scale=2):
+            model_name = gr.Textbox(label="模型名称（必填）", placeholder="例如：my-agent-v1")
+            model_api = gr.Textbox(
+                label="模型 API（必填）",
+                placeholder="例如：https://api.example.com/v1/chat/completions",
+            )
+            notes = gr.Textbox(label="备注（可选）", lines=4)
+
+            # 纯前端版：不强制 OAuth；在 Space 里建议你自己加 LoginButton 做鉴权
+            if IN_SPACES:
+                username = gr.Textbox(
+                    label="用户名（可选）",
+                    placeholder="建议填你的 HF 用户名（也可留空）",
+                )
+            else:
+                username = gr.Textbox(label="用户名（本地调试用）", value="local")
+
+            submit_btn = gr.Button("提交", variant="primary")
+            status = gr.Markdown()
+        with gr.Column(scale=3):
+            leaderboard = gr.Dataframe(
+                label="Leaderboard（按提交时间倒序）",
+                value=_load_submissions_df(),
+                interactive=False,
+                wrap=True,
+            )
+            refresh_btn = gr.Button("刷新")
+
+    submit_btn.click(
+        submit,
+        inputs=[model_name, model_api, notes, username],
+        outputs=[status, leaderboard],
+    )
+    refresh_btn.click(refresh, inputs=[], outputs=[leaderboard])
+
+def main():
+    demo.launch()
+
+
+if __name__ == "__main__":
+    main()
+
+# Display the results
+if HAS_TOKEN and not LOCAL_DEBUG:
+    try:
+        eval_results = load_dataset(
+            RESULTS_DATASET,
+            YEAR_VERSION,
+            token=TOKEN,
+            download_mode="force_redownload",
+            verification_mode=VerificationMode.NO_CHECKS,
+        )
+    except Exception as e:
+        print(e)
+        eval_results = None
+
+    try:
+        contact_infos = load_dataset(
+            CONTACT_DATASET,
+            YEAR_VERSION,
+            token=TOKEN,
+            download_mode="force_redownload",
+            verification_mode=VerificationMode.NO_CHECKS,
+        )
+    except Exception as e:
+        print(e)
+        contact_infos = None
+else:
+    eval_results = None
+    contact_infos = None
+
+def get_dataframe_from_results(eval_results, split):
+    if eval_results is None:
+        return pd.DataFrame(columns=EMPTY_LEADERBOARD_COLUMNS)
+    local_df = eval_results[split]
+    local_df = local_df.map(lambda row: {"model": model_hyperlink(row["url"], row["model"])})
+    local_df = local_df.remove_columns(["system_prompt", "url"])
+    local_df = local_df.rename_column("model", "Agent name")
+    local_df = local_df.rename_column("model_family", "Model family")
+    local_df = local_df.rename_column("score", "Average score (%)")
+    for i in [1, 2, 3]:
+        local_df = local_df.rename_column(f"score_level{i}", f"Level {i} score (%)")
+    local_df = local_df.rename_column("date", "Submission date")
+    df = pd.DataFrame(local_df)
+    df = df.sort_values(by=["Average score (%)"], ascending=False)
+
+    numeric_cols = [c for c in local_df.column_names if "score" in c]
+    df[numeric_cols] = df[numeric_cols].multiply(100).round(decimals=2)
+    #df = df.style.format("{:.2%}", subset=numeric_cols)
+
+    return df
+
+#eval_dataframe_val = get_dataframe_from_results(eval_results=eval_results, split="validation")
+eval_dataframe_test = get_dataframe_from_results(eval_results=eval_results, split="test")
+
+# Gold answers
+if HAS_TOKEN and not LOCAL_DEBUG:
+    gold_dataset = load_dataset(
+        INTERNAL_DATA_DATASET,
+        f"{YEAR_VERSION}_all",
+        token=TOKEN,
+    )
+    gold_results = {
+        split: {row["task_id"]: row for row in gold_dataset[split]}
+        for split in ["test", "validation"]
+    }
+else:
+    gold_results = {"test": {}, "validation": {}}
+
+
+def restart_space():
+    if IN_SPACES and HAS_TOKEN:
+        api.restart_space(repo_id=LEADERBOARD_PATH, token=TOKEN)
+
+TYPES = ["markdown", "number", "number", "number", "number", "str", "str", "str"]
+
+def add_new_eval(
+    #val_or_test: str,
+    model: str,
+    model_family: str,
+    system_prompt: str,
+    url: str,
+    path_to_file: str,
+    organisation: str,
+    mail: str,
+    profile: gr.OAuthProfile, 
+):
+    val_or_test = "test"
+    try:
+        if not HAS_TOKEN or LOCAL_DEBUG:
+            return format_error(
+                "Submissions are disabled in local mode. Set env TOKEN (Hugging Face token) and rerun to enable submissions."
+            )
+        # Was the profile created less than 2 month ago?
+        user_data = requests.get(f"https://huggingface.co/api/users/{profile.username}/overview")
+        creation_date = json.loads(user_data.content)["createdAt"]
+        if datetime.datetime.now() - datetime.datetime.strptime(creation_date, '%Y-%m-%dT%H:%M:%S.%fZ') < datetime.timedelta(days=60):
+            return format_error("This account is not authorized to submit on GAIA.")
+            
+
+        contact_infos = load_dataset(CONTACT_DATASET, YEAR_VERSION, token=TOKEN, download_mode="force_redownload", verification_mode=VerificationMode.NO_CHECKS, trust_remote_code=True)
+        user_submission_dates = sorted(row["date"] for row in contact_infos[val_or_test] if row["username"] == profile.username)
+        if len(user_submission_dates) > 0 and user_submission_dates[-1] == datetime.datetime.today().strftime('%Y-%m-%d'):
+            return format_error("You already submitted once today, please try again tomorrow.")
+
+
+        is_validation = val_or_test == "validation"
+        # Very basic email parsing
+        _, parsed_mail = parseaddr(mail)
+        if not "@" in parsed_mail:
+            return format_warning("Please provide a valid email adress.")
+
+        print("Adding new eval")
+
+        # Check if the combination model/org already exists and prints a warning message if yes
+        if model.lower() in set([m.lower() for m in eval_results[val_or_test]["model"]]) and organisation.lower() in set([o.lower() for o in eval_results[val_or_test]["organisation"]]):
+            return format_warning("This model has been already submitted.")
+        
+        if path_to_file is None:
+            return format_warning("Please attach a file.")
+
+        # SAVE UNSCORED SUBMISSION
+        if LOCAL_DEBUG:
+            print("mock uploaded submission")
+        else:
+            api.upload_file(
+                repo_id=SUBMISSION_DATASET, 
+                path_or_fileobj=path_to_file.name, 
+                path_in_repo=f"{organisation}/{model}/{YEAR_VERSION}_{val_or_test}_raw_{datetime.datetime.today()}.jsonl",
+                repo_type="dataset", 
+                token=TOKEN
+            )
+
+        # SAVE CONTACT
+        contact_info = {
+            "model": model,
+            "model_family": model_family,
+            "url": url,
+            "organisation": organisation,
+            "username": profile.username,
+            "mail": mail,
+            "date": datetime.datetime.today().strftime('%Y-%m-%d')
+        }
+        contact_infos[val_or_test]= contact_infos[val_or_test].add_item(contact_info)
+        if LOCAL_DEBUG:
+            print("mock uploaded contact info")
+        else:
+            contact_infos.push_to_hub(CONTACT_DATASET, config_name = YEAR_VERSION, token=TOKEN)
+
+        # SCORE SUBMISSION
+        file_path = path_to_file.name        
+        scores = {"all": 0, 1: 0, 2: 0, 3: 0}
+        num_questions = {"all": 0, 1: 0, 2: 0, 3: 0}
+        task_ids = []
+        with open(f"scored/{organisation}_{model}.jsonl", "w") as scored_file:
+            with open(file_path, 'r') as f:
+                for ix, line in enumerate(f):
+                    try:
+                        task = json.loads(line)
+                    except Exception:
+                        return format_error(f"Line {ix} is incorrectly formatted. Please fix it and resubmit your file.")
+
+                    if "model_answer" not in task:
+                        return format_error(f"Line {ix} contains no model_answer key. Please fix it and resubmit your file.")
+                    answer = task["model_answer"]
+                    task_id = task["task_id"]
+                    try:
+                        level = int(gold_results[val_or_test][task_id]["Level"])
+                    except KeyError:
+                        return format_error(f"{task_id} not found in split {val_or_test}. Are you sure you submitted the correct file?")
+
+                    score = question_scorer(task['model_answer'], gold_results[val_or_test][task_id]["Final answer"])
+                    
+                    scored_file.write(
+                        json.dumps({
+                            "id": task_id,
+                            "model_answer": answer,
+                            "score": score,
+                            "level": level
+                        }) + "\n"
+                    )
+                    task_ids.append(task_id)
+
+                    scores["all"] += score
+                    scores[level] += score
+                    num_questions["all"] += 1
+                    num_questions[level] += 1
+
+        # Check if there's any duplicate in the submission
+        if len(task_ids) != len(set(task_ids)):
+            return format_error("There are duplicates in your submission. Please check your file and resubmit it.")
+
+        if any([num_questions[level] != ref_level_len[val_or_test][level] for level in [1, 2, 3]]):
+            return format_error(f"Your submission has {num_questions[1]} questions for level 1, {num_questions[2]} for level 2, and {num_questions[3]} for level 3, but it should have {ref_level_len[val_or_test][1]}, {ref_level_len[val_or_test][2]}, and {ref_level_len[val_or_test][3]} respectively. Please check your submission.")
+
+        # SAVE SCORED SUBMISSION
+        if LOCAL_DEBUG:
+            print("mock uploaded scored submission")
+        else:
+            api.upload_file(
+                repo_id=SUBMISSION_DATASET, 
+                path_or_fileobj=f"scored/{organisation}_{model}.jsonl",
+                path_in_repo=f"{organisation}/{model}/{YEAR_VERSION}_{val_or_test}_scored_{datetime.datetime.today()}.jsonl", 
+                repo_type="dataset", 
+                token=TOKEN
+            )
+
+            # Save scored file
+            if is_validation:
+                api.upload_file(
+                    repo_id=SUBMISSION_DATASET_PUBLIC, 
+                    path_or_fileobj=f"scored/{organisation}_{model}.jsonl",
+                    path_in_repo=f"{organisation}/{model}/{YEAR_VERSION}_{val_or_test}_scored_{datetime.datetime.today()}.jsonl", 
+                    repo_type="dataset", 
+                    token=TOKEN
+                )
+
+        # SAVE TO LEADERBOARD DATA
+        eval_entry = {
+            "model": model,
+            "model_family": model_family,
+            "system_prompt": system_prompt,
+            "url": url,
+            "organisation": organisation,
+            "score": scores["all"]/ref_scores_len[val_or_test],
+            "score_level1": scores[1]/num_questions[1],
+            "score_level2": scores[2]/num_questions[2],
+            "score_level3": scores[3]/num_questions[3],
+            "date": datetime.datetime.today().strftime('%Y-%m-%d')
+        }
+        if num_questions[1] + num_questions[2] + num_questions[3] != ref_scores_len[val_or_test]:
+            return format_error(f"Your submission has {len(scores['all'])} questions for the {val_or_test} set, but it should have {ref_scores_len[val_or_test]}. Please check your submission.")
+        # Catching spam submissions of 100%
+        if all((eval_entry[k] == 1 for k in ["score_level1", "score_level2", "score_level3"])):
+            return format_error(f"There was a problem with your submission. Please open a discussion.")
+
+        # Testing for duplicates - to see if we want to add something like it as it would allow people to try to see the content of other submissions
+        #eval_entry_no_date = {k: v for k, v in eval_entry if k != "date"}
+        #columns_no_date = [c for c in eval_results[val_or_test].column_names if c != "date"]
+        #if eval_entry_no_date in eval_results[val_or_test].select_columns(columns_no_date):
+        #    return format_error(f"Your submission is an exact duplicate from an existing submission.")
+
+        eval_results[val_or_test] = eval_results[val_or_test].add_item(eval_entry)
+        print(eval_results)
+        if LOCAL_DEBUG:
+            print("mock uploaded results to lb")
+        else:
+            eval_results.push_to_hub(RESULTS_DATASET, config_name = YEAR_VERSION, token=TOKEN)
+
+
+        return format_log(f"Model {model} submitted by {organisation} successfully.\nPlease wait a few hours and refresh the leaderboard to see your score displayed.")
+    except Exception as e:
+        print(e)
+        return format_error(f"An error occurred, please open a discussion and indicate at what time you encountered the error.\n")
+
+
+def refresh():
+    if HAS_TOKEN and not LOCAL_DEBUG:
+        try:
+            eval_results = load_dataset(
+                RESULTS_DATASET,
+                YEAR_VERSION,
+                token=TOKEN,
+                download_mode="force_redownload",
+                verification_mode=VerificationMode.NO_CHECKS,
+            )
+        except Exception as e:
+            print(e)
+            eval_results = None
+    else:
+        eval_results = None
+    return get_dataframe_from_results(eval_results=eval_results, split="test")
+
+def upload_file(files):
+    file_paths = [file.name for file in files]
+    return file_paths
+
+
+demo = gr.Blocks()
+with demo:
+    gr.HTML(TITLE)
+    gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
+
+    with gr.Row():
+        with gr.Accordion("📙 Citation", open=False):
+            citation_button = gr.Textbox(
+                value=CITATION_BUTTON_TEXT,
+                label=CITATION_BUTTON_LABEL,
+                elem_id="citation-button",
+            ) #.style(show_copy_button=True)
+
+    gr.Markdown("Results: Test")
+    leaderboard_table_test = gr.components.Dataframe(
+        value=eval_dataframe_test, datatype=TYPES, interactive=False,
+        column_widths=["20%"] 
+    )
+    #with gr.Tab("Results: Validation"):
+    #    leaderboard_table_val = gr.components.Dataframe(
+    #        value=eval_dataframe_val, datatype=TYPES, interactive=False,
+    #        column_widths=["20%"] 
+    #    )
+
+    refresh_button = gr.Button("Refresh")
+    refresh_button.click(
+        refresh,
+        inputs=[],
+        outputs=[
+            #leaderboard_table_val,
+            leaderboard_table_test,
+        ],
+    )
+    with gr.Accordion("Submit a new model for evaluation"):
+        with gr.Row():
+            gr.Markdown(SUBMISSION_TEXT, elem_classes="markdown-text")
+        with gr.Row():
+            with gr.Column():
+                #level_of_test = gr.Radio(["test"], value="test", label="Split")
+                model_name_textbox = gr.Textbox(label="Agent name")
+                model_family_textbox = gr.Textbox(label="Model family")
+                system_prompt_textbox = gr.Textbox(label="System prompt example")
+                url_textbox = gr.Textbox(label="Url to model information")
+            with gr.Column():
+                organisation = gr.Textbox(label="Organisation")
+                mail = gr.Textbox(label="Contact email (will be stored privately, & used if there is an issue with your submission)")
+                file_output = gr.File()
+
+
+        with gr.Row():
+            gr.LoginButton()
+            submit_button = gr.Button("Submit Eval On Test")
+        submission_result = gr.Markdown()
+        submit_button.click(
+            add_new_eval,
+            [
+                #level_of_test,
+                model_name_textbox,
+                model_family_textbox,
+                system_prompt_textbox,
+                url_textbox,
+                file_output,
+                organisation,
+                mail
+            ],
+            submission_result,
+        )
+
+if IN_SPACES and HAS_TOKEN:
+    scheduler = BackgroundScheduler()
+    scheduler.add_job(restart_space, "interval", seconds=3600)
+    scheduler.start()
+demo.launch(debug=True)

content.py

@@ -0,0 +1,65 @@
+TITLE = """<h1 align="center" id="space-title">GAIA Leaderboard</h1>"""
+
+INTRODUCTION_TEXT = """
+GAIA is a benchmark which aims at evaluating next-generation LLMs (LLMs with augmented capabilities due to added tooling, efficient prompting, access to search, etc). (See our [paper](https://arxiv.org/abs/2311.12983) for more details.)
+
+## Data
+GAIA is made of more than 450 non-trivial question with an unambiguous answer, requiring different levels of tooling and autonomy to solve. 
+It is therefore divided in 3 levels, where level 1 should be breakable by very good LLMs, and level 3 indicate a strong jump in model capabilities. Each level is divided into a fully public dev set for validation, and a test set with private answers and metadata. 
+
+GAIA data can be found in [this dataset](https://huggingface.co/datasets/gaia-benchmark/GAIA). Questions are contained in `metadata.jsonl`. Some questions come with an additional file, that can be found in the same folder and whose id is given in the field `file_name`.
+
+**Please do not repost the public dev set, nor use it in training data for your models.**
+
+## Leaderboard
+Submission made by our team are labelled "GAIA authors". While we report average scores over different runs when possible in our paper, we only report the best run in the leaderboard.
+
+See below for submissions.
+"""
+
+SUBMISSION_TEXT = """
+## Submissions
+Results can be submitted for the test set (we closed the validation leaderboard, as it was no longer informative). Scores are expressed as the percentage of correct answers for a given split. 
+
+Each question calls for an answer that is either a string (one or a few words), a number, or a comma separated list of strings or floats, unless specified otherwise. There is only one correct answer. 
+Hence, evaluation is done via quasi exact match between a model’s answer and the ground truth (up to some normalization that is tied to the “type” of the ground truth).
+
+In our evaluation, we use a system prompt to instruct the model about the required format:
+```
+You are a general AI assistant. I will ask you a question. Report your thoughts, and finish your answer with the following template: FINAL ANSWER: [YOUR FINAL ANSWER]. YOUR FINAL ANSWER should be a number OR as few words as possible OR a comma separated list of numbers and/or strings. If you are asked for a number, don't use comma to write your number neither use units such as $ or percent sign unless specified otherwise. If you are asked for a string, don't use articles, neither abbreviations (e.g. for cities), and write the digits in plain text unless specified otherwise. If you are asked for a comma separated list, apply the above rules depending of whether the element to be put in the list is a number or a string.
+```
+We advise you to use the system prompt provided in the paper to ensure your agents answer using the correct and expected format. In practice, GPT4 level models easily follow it.
+
+
+We expect submissions to be json-line files with the following format. The first two fields are mandatory, `reasoning_trace` is optional:
+```
+{"task_id": "task_id_1", "model_answer": "Answer 1 from your model", "reasoning_trace": "The different steps by which your model reached answer 1"}
+{"task_id": "task_id_2", "model_answer": "Answer 2 from your model", "reasoning_trace": "The different steps by which your model reached answer 2"}
+```
+
+Our scoring function can be found [here](https://huggingface.co/spaces/gaia-benchmark/leaderboard/blob/main/scorer.py).
+"""
+
+CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results"
+CITATION_BUTTON_TEXT = r"""@misc{mialon2023gaia,
+      title={GAIA: a benchmark for General AI Assistants}, 
+      author={Grégoire Mialon and Clémentine Fourrier and Craig Swift and Thomas Wolf and Yann LeCun and Thomas Scialom},
+      year={2023},
+      eprint={2311.12983},
+      archivePrefix={arXiv},
+      primaryClass={cs.CL}
+}"""
+
+
+def format_error(msg):
+    return f"<p style='color: red; font-size: 20px; text-align: center;'>{msg}</p>"
+
+def format_warning(msg):
+    return f"<p style='color: orange; font-size: 20px; text-align: center;'>{msg}</p>"
+
+def format_log(msg):
+    return f"<p style='color: green; font-size: 20px; text-align: center;'>{msg}</p>"
+
+def model_hyperlink(link, model_name):
+    return f'<a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{model_name}</a>'
+

gitattributes

@@ -0,0 +1,34 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

gitignore

@@ -0,0 +1,4 @@
+.vscode
+__pycache__/*
+# Local results
+scored/*

miniapp.py

@@ -0,0 +1,71 @@
+import datetime
+from urllib.parse import urlparse
+
+import gradio as gr
+
+
+APP_NAME = "miniapp"
+
+
+def _is_valid_http_url(url: str) -> bool:
+    try:
+        parsed = urlparse(url)
+        return parsed.scheme in ("http", "https") and bool(parsed.netloc)
+    except Exception:
+        return False
+
+
+def submit(model_name: str, model_api: str, notes: str):
+    model_name = (model_name or "").strip()
+    model_api = (model_api or "").strip()
+    notes = (notes or "").strip()
+
+    if not model_name:
+        return "请填写 **模型名称**。", None
+    if not model_api:
+        return "请填写 **模型 API**。", None
+    if not _is_valid_http_url(model_api):
+        return "**模型 API** 需要是合法的 `http(s)://...` URL。", None
+
+    payload = {
+        "model_name": model_name,
+        "model_api": model_api,
+        "notes": notes,
+        "submitted_at": datetime.datetime.now().isoformat(timespec="seconds"),
+    }
+    return "已收到提交（仅前端回显；未做评测/未写入排行榜）。", payload
+
+
+with gr.Blocks(title=APP_NAME) as demo:
+    gr.Markdown(
+        f"## {APP_NAME}\n\n"
+        "纯前端信息收集页：填写模型名称与 API 地址，点击提交后回显。\n\n"
+        "- 不需要 Hugging Face 登录\n"
+        "- 不依赖 scorer\n"
+        "- 不读写任何 leaderboard 数据\n"
+    )
+
+    with gr.Row():
+        with gr.Column(scale=2):
+            model_name = gr.Textbox(label="模型名称（必填）", placeholder="例如：my-agent-v1")
+            model_api = gr.Textbox(
+                label="模型 API（必填）",
+                placeholder="例如：https://api.example.com/v1/chat/completions",
+            )
+            notes = gr.Textbox(
+                label="备注（可选）",
+                lines=4,
+                placeholder="例如：鉴权方式、限流说明、模型简介等",
+            )
+            submit_btn = gr.Button("提交", variant="primary")
+        with gr.Column(scale=3):
+            status = gr.Markdown()
+            submission_json = gr.JSON(label="提交内容（回显）")
+
+    submit_btn.click(
+        submit,
+        inputs=[model_name, model_api, notes],
+        outputs=[status, submission_json],
+    )
+
+demo.launch()

miniapp_leaderboard.py

@@ -0,0 +1,728 @@
+import datetime
+import io
+import json
+import os
+import re
+import uuid
+from urllib.parse import urlparse
+
+import gradio as gr
+import pandas as pd
+import requests
+from huggingface_hub import HfApi, hf_hub_download
+from huggingface_hub.errors import RepositoryNotFoundError
+
+
+APP_NAME = "miniapp"
+
+# Space Secrets:
+# - HF_TOKEN: Hugging Face access token with WRITE access to the dataset repo
+# - LEADERBOARD_DATASET: dataset repo id like "ha251/miniapp-leaderboard"
+HF_TOKEN = os.environ.get("HF_TOKEN") or os.environ.get("TOKEN") or os.environ.get("HUGGINGFACE_TOKEN")
+LEADERBOARD_DATASET = (os.environ.get("LEADERBOARD_DATASET") or "").strip()
+MAX_ENTRIES = int(os.environ.get("MAX_ENTRIES", "500"))
+
+ENTRIES_PREFIX = "entries/"
+
+LEADERBOARD_COLUMNS = [
+    "Model name",
+    "Avg",
+    "Easy",
+    "Mid",
+    "Hard",
+    "Games",
+    "Science",
+    "Tools",
+    "Humanities",
+    "Viz",
+    "Lifestyle",
+    "Submitted at",
+    "Submitter",
+]
+
+NUMERIC_COLS = [
+    "Avg",
+    "Easy",
+    "Mid",
+    "Hard",
+    "Games",
+    "Science",
+    "Tools",
+    "Humanities",
+    "Viz",
+    "Lifestyle",
+]
+
+IN_SPACES = bool(
+    os.environ.get("SPACE_ID")
+    or os.environ.get("SPACE_REPO_NAME")
+    or os.environ.get("SPACE_AUTHOR_NAME")
+    or os.environ.get("system", "") == "spaces"
+)
+
+
+def _api() -> HfApi:
+    return HfApi(token=HF_TOKEN)
+
+
+def _is_valid_http_url(url: str) -> bool:
+    try:
+        parsed = urlparse(url)
+        return parsed.scheme in ("http", "https") and bool(parsed.netloc)
+    except Exception:
+        return False
+
+
+def _slug(s: str, max_len: int = 60) -> str:
+    s = (s or "").strip().lower()
+    s = re.sub(r"[^a-z0-9]+", "-", s)
+    s = re.sub(r"-{2,}", "-", s).strip("-")
+    return (s[:max_len] or "x")
+
+
+def _empty_df() -> pd.DataFrame:
+    return pd.DataFrame(columns=LEADERBOARD_COLUMNS)
+
+
+def _ensure_dataset_readable() -> tuple[bool, str]:
+    if not HF_TOKEN:
+        return False, "Space is missing HF_TOKEN (Secrets)."
+    if not LEADERBOARD_DATASET:
+        return False, "Space is missing LEADERBOARD_DATASET (Secrets)."
+    api = _api()
+    try:
+        api.repo_info(repo_id=LEADERBOARD_DATASET, repo_type="dataset")
+        return True, ""
+    except RepositoryNotFoundError:
+        return False, (
+            f"Dataset repo not found: {LEADERBOARD_DATASET}. "
+            "Create it first (as a dataset) or fix LEADERBOARD_DATASET."
+        )
+    except Exception:
+        return False, "Cannot access the dataset repo. Check token permissions."
+
+
+def _list_entry_files() -> list[str]:
+    ok, _ = _ensure_dataset_readable()
+    if not ok:
+        return []
+    api = _api()
+    try:
+        files = api.list_repo_files(repo_id=LEADERBOARD_DATASET, repo_type="dataset")
+    except Exception:
+        return []
+
+    entry_files = [f for f in files if f.startswith(ENTRIES_PREFIX) and f.endswith(".json")]
+    entry_files.sort(reverse=True)
+    return entry_files[:MAX_ENTRIES]
+
+
+def _load_entries_df() -> pd.DataFrame:
+    ok, _ = _ensure_dataset_readable()
+    if not ok:
+        return _empty_df()
+
+    rows: list[dict] = []
+    for filename in _list_entry_files():
+        try:
+            path = hf_hub_download(
+                repo_id=LEADERBOARD_DATASET,
+                repo_type="dataset",
+                filename=filename,
+                token=HF_TOKEN,
+            )
+            with open(path, "r", encoding="utf-8") as fp:
+                row = json.load(fp)
+                rows.append(row)
+        except Exception:
+            continue
+
+    if not rows:
+        return _empty_df()
+
+    df = pd.DataFrame(rows)
+    for c in LEADERBOARD_COLUMNS:
+        if c not in df.columns:
+            df[c] = ""
+    df = df[LEADERBOARD_COLUMNS]
+    for c in NUMERIC_COLS:
+        df[c] = pd.to_numeric(df[c], errors="coerce")
+    df = df.sort_values(by=["Submitted at"], ascending=False, kind="stable")
+    return df
+
+
+def _apply_search_and_sort(
+    df: pd.DataFrame,
+    search_text: str,
+    search_in: str,
+    sort_by: str,
+    sort_order: str,
+) -> pd.DataFrame:
+    search_text = (search_text or "").strip().lower()
+    if search_text:
+        if search_in == "Model name":
+            df = df[df["Model name"].astype(str).str.lower().str.contains(search_text, na=False)]
+        elif search_in == "Submitter":
+            df = df[df["Submitter"].astype(str).str.lower().str.contains(search_text, na=False)]
+        else:
+            mask = (
+                df["Model name"].astype(str).str.lower().str.contains(search_text, na=False)
+                | df["Submitter"].astype(str).str.lower().str.contains(search_text, na=False)
+            )
+            df = df[mask]
+
+    ascending = sort_order == "Ascending"
+    if sort_by in df.columns:
+        df = df.sort_values(by=[sort_by], ascending=ascending, kind="stable", na_position="last")
+    return df
+
+
+def refresh(search_text: str, search_in: str, sort_by: str, sort_order: str):
+    df = _load_entries_df()
+    return _apply_search_and_sort(df, search_text, search_in, sort_by, sort_order)
+
+
+def _parse_hf_created_at(created_at: str) -> datetime.datetime | None:
+    try:
+        if created_at.endswith("Z"):
+            created_at = created_at[:-1] + "+00:00"
+        return datetime.datetime.fromisoformat(created_at)
+    except Exception:
+        return None
+
+
+def _check_user_eligibility(username: str) -> tuple[bool, str]:
+    """
+    - Must be older than ~4 months (>= 120 days)
+    """
+    try:
+        r = requests.get(f"https://huggingface.co/api/users/{username}/overview", timeout=10)
+        r.raise_for_status()
+        created_at = r.json().get("createdAt")
+        if not created_at:
+            return False, "Cannot verify account creation date."
+        dt = _parse_hf_created_at(created_at)
+        if not dt:
+            return False, "Cannot parse account creation date."
+        now = datetime.datetime.now(datetime.timezone.utc)
+        if dt.tzinfo is None:
+            dt = dt.replace(tzinfo=datetime.timezone.utc)
+        age_days = (now - dt).days
+        if age_days < 120:
+            return False, "Account must be older than 4 months to submit."
+        return True, ""
+    except Exception:
+        return False, "Cannot verify Hugging Face account. Please try again later."
+
+
+def _submitted_today(username: str) -> bool:
+    df = _load_entries_df()
+    if df.empty:
+        return False
+    today = datetime.datetime.utcnow().date().isoformat()
+    user_rows = df[df["Submitter"].astype(str) == username]
+    if user_rows.empty:
+        return False
+    return any(str(v).startswith(today) for v in user_rows["Submitted at"].tolist())
+
+
+def submit(
+    model_name: str,
+    model_api: str,
+    api_key: str,
+    avg: float,
+    easy: float,
+    mid: float,
+    hard: float,
+    games: float,
+    science: float,
+    tools: float,
+    humanities: float,
+    viz: float,
+    lifestyle: float,
+    search_text: str,
+    search_in: str,
+    sort_by: str,
+    sort_order: str,
+    profile: gr.OAuthProfile | None,
+):
+    # Login required to submit (in Spaces)
+    if IN_SPACES and (profile is None or not getattr(profile, "username", None)):
+        return "You must log in to submit.", refresh(search_text, search_in, sort_by, sort_order)
+
+    submitter = (getattr(profile, "username", None) if profile is not None else "local") or "anonymous"
+
+    model_name = (model_name or "").strip()
+    model_api = (model_api or "").strip()
+    api_key = (api_key or "").strip()
+
+    if not model_name:
+        return "Model name is required.", refresh(search_text, search_in, sort_by, sort_order)
+    if not model_api:
+        return "Model API URL is required.", refresh(search_text, search_in, sort_by, sort_order)
+    if not _is_valid_http_url(model_api):
+        return "Model API must be a valid http(s) URL.", refresh(search_text, search_in, sort_by, sort_order)
+    if not api_key:
+        return "API key is required.", refresh(search_text, search_in, sort_by, sort_order)
+
+    ok, msg = _ensure_dataset_readable()
+    if not ok:
+        return msg, refresh(search_text, search_in, sort_by, sort_order)
+
+    if IN_SPACES:
+        ok, msg = _check_user_eligibility(submitter)
+        if not ok:
+            return msg, refresh(search_text, search_in, sort_by, sort_order)
+
+    if _submitted_today(submitter):
+        return "You have already submitted today. Please try again tomorrow.", refresh(search_text, search_in, sort_by, sort_order)
+
+    now = datetime.datetime.utcnow().replace(microsecond=0).isoformat() + "Z"
+    nonce = uuid.uuid4().hex[:8]
+    safe_model = _slug(model_name)
+    safe_user = _slug(submitter)
+    path_in_repo = f"{ENTRIES_PREFIX}{now[:10]}/{now}-{safe_user}-{safe_model}-{nonce}.json"
+
+    # NOTE: api_key is collected but NOT stored.
+    payload = {
+        "Model name": model_name,
+        "Avg": avg,
+        "Easy": easy,
+        "Mid": mid,
+        "Hard": hard,
+        "Games": games,
+        "Science": science,
+        "Tools": tools,
+        "Humanities": humanities,
+        "Viz": viz,
+        "Lifestyle": lifestyle,
+        "Submitted at": now,
+        "Submitter": submitter,
+        "Model API": model_api,
+    }
+
+    api = _api()
+    data = (json.dumps(payload, ensure_ascii=False, indent=2) + "\n").encode("utf-8")
+    api.upload_file(
+        repo_id=LEADERBOARD_DATASET,
+        repo_type="dataset",
+        path_or_fileobj=io.BytesIO(data),
+        path_in_repo=path_in_repo,
+        commit_message=f"miniapp: submit {submitter}/{model_name}",
+        token=HF_TOKEN,
+    )
+
+    return "Submitted successfully.", refresh(search_text, search_in, sort_by, sort_order)
+
+
+with gr.Blocks(title=f"{APP_NAME} leaderboard") as demo:
+    # Layout: Overview -> Leaderboard -> Submission instructions
+    gr.Markdown(
+        f"## {APP_NAME} leaderboard\n\n"
+        "### Overview\n"
+        "_Placeholder overview text. Replace this with your benchmark description, rules, and links._\n\n"
+        "### Leaderboard\n"
+        "_The leaderboard below supports sorting and searching._\n\n"
+        "### Submission\n"
+        "_Placeholder submission instructions. Requires login. One submission per user per day. "
+        "Account must be older than 4 months._\n"
+    )
+
+    with gr.Row():
+        with gr.Column(scale=3):
+            with gr.Row():
+                search_text = gr.Textbox(label="Search", placeholder="Model name or submitter")
+                search_in = gr.Dropdown(
+                    label="Search in",
+                    choices=["Both", "Model name", "Submitter"],
+                    value="Both",
+                )
+            with gr.Row():
+                sort_by = gr.Dropdown(label="Sort by", choices=LEADERBOARD_COLUMNS, value="Avg")
+                sort_order = gr.Radio(label="Order", choices=["Descending", "Ascending"], value="Descending")
+                refresh_btn = gr.Button("Refresh")
+
+            leaderboard = gr.Dataframe(
+                label="Leaderboard",
+                value=_load_entries_df(),
+                interactive=False,
+                wrap=True,
+            )
+
+        with gr.Column(scale=2):
+            with gr.Accordion("Submit (login required)", open=True):
+                model_name = gr.Textbox(label="Model name (required)")
+                model_api = gr.Textbox(label="Model API (required)", placeholder="https://...")
+                api_key = gr.Textbox(label="API key (required)", type="password", placeholder="Will not be stored")
+
+                with gr.Row():
+                    avg = gr.Number(label="Avg", value=0)
+                    easy = gr.Number(label="Easy", value=0)
+                    mid = gr.Number(label="Mid", value=0)
+                    hard = gr.Number(label="Hard", value=0)
+
+                with gr.Row():
+                    games = gr.Number(label="Games", value=0)
+                    science = gr.Number(label="Science", value=0)
+                    tools = gr.Number(label="Tools", value=0)
+
+                with gr.Row():
+                    humanities = gr.Number(label="Humanities", value=0)
+                    viz = gr.Number(label="Viz", value=0)
+                    lifestyle = gr.Number(label="Lifestyle", value=0)
+
+                with gr.Row():
+                    gr.LoginButton()
+                    submit_btn = gr.Button("Submit", variant="primary")
+                status = gr.Markdown()
+
+    refresh_btn.click(refresh, inputs=[search_text, search_in, sort_by, sort_order], outputs=[leaderboard])
+    submit_btn.click(
+        submit,
+        inputs=[
+            model_name,
+            model_api,
+            api_key,
+            avg,
+            easy,
+            mid,
+            hard,
+            games,
+            science,
+            tools,
+            humanities,
+            viz,
+            lifestyle,
+            search_text,
+            search_in,
+            sort_by,
+            sort_order,
+        ],
+        outputs=[status, leaderboard],
+    )
+
+demo.launch()
+
+import datetime
+import io
+import json
+import os
+import re
+import uuid
+from urllib.parse import urlparse
+
+import gradio as gr
+import pandas as pd
+from huggingface_hub import HfApi, hf_hub_download
+
+
+APP_NAME = "miniapp"
+
+# 在 Space 里通过 Secrets 配置：
+# - HF_TOKEN: 具有写 dataset 权限的 token（Settings -> Variables and secrets -> Secrets）
+# - LEADERBOARD_DATASET: 形如 "your-username/miniapp-leaderboard"（repo_type=dataset）
+HF_TOKEN = os.environ.get("HF_TOKEN") or os.environ.get("TOKEN") or os.environ.get("HUGGINGFACE_TOKEN")
+LEADERBOARD_DATASET = os.environ.get("LEADERBOARD_DATASET", "").strip()
+# Owner 审核口令（放到 Space Secrets；不要放到公开 Variables）
+OWNER_REVIEW_TOKEN = os.environ.get("OWNER_REVIEW_TOKEN", "").strip()
+
+# 判断是否运行在 Hugging Face Spaces
+IN_SPACES = bool(
+    os.environ.get("SPACE_ID")
+    or os.environ.get("SPACE_REPO_NAME")
+    or os.environ.get("SPACE_AUTHOR_NAME")
+    or os.environ.get("system", "") == "spaces"
+)
+
+MAX_ENTRIES = int(os.environ.get("MAX_ENTRIES", "200"))
+
+PENDING_PREFIX = "pending/"
+APPROVED_PREFIX = "approved/"
+
+
+def _is_valid_http_url(url: str) -> bool:
+    try:
+        parsed = urlparse(url)
+        return parsed.scheme in ("http", "https") and bool(parsed.netloc)
+    except Exception:
+        return False
+
+
+def _slug(s: str, max_len: int = 60) -> str:
+    s = (s or "").strip().lower()
+    s = re.sub(r"[^a-z0-9]+", "-", s)
+    s = re.sub(r"-{2,}", "-", s).strip("-")
+    return (s[:max_len] or "model")
+
+
+def _api() -> HfApi:
+    return HfApi(token=HF_TOKEN)
+
+
+def _ensure_dataset_repo():
+    if not HF_TOKEN:
+        raise RuntimeError("未配置 HF_TOKEN（Space Secrets）。")
+    if not LEADERBOARD_DATASET:
+        raise RuntimeError("未配置 LEADERBOARD_DATASET（例如：your-username/miniapp-leaderboard）。")
+    api = _api()
+    try:
+        api.repo_info(repo_id=LEADERBOARD_DATASET, repo_type="dataset")
+    except Exception:
+        # 不存在则创建（public dataset；你也可以手动创建并设为 private）
+        api.create_repo(repo_id=LEADERBOARD_DATASET, repo_type="dataset", private=False, exist_ok=True)
+
+
+def _empty_df() -> pd.DataFrame:
+    return pd.DataFrame(columns=["submitted_at", "username", "model_name", "model_api", "notes"])
+
+
+def _list_json_files(prefix: str) -> list[str]:
+    if not HF_TOKEN or not LEADERBOARD_DATASET:
+        return []
+
+    api = _api()
+    try:
+        files = api.list_repo_files(repo_id=LEADERBOARD_DATASET, repo_type="dataset")
+    except Exception:
+        return []
+
+    return sorted(
+        [f for f in files if f.startswith(prefix) and f.endswith(".json")],
+        reverse=True,
+    )[:MAX_ENTRIES]
+
+
+def _load_entries_df(prefix: str, include_filename: bool) -> pd.DataFrame:
+    files = _list_json_files(prefix)
+    rows = []
+    for filename in files:
+        try:
+            path = hf_hub_download(
+                repo_id=LEADERBOARD_DATASET,
+                repo_type="dataset",
+                filename=filename,
+                token=HF_TOKEN,
+            )
+            with open(path, "r", encoding="utf-8") as fp:
+                row = json.load(fp)
+                if include_filename:
+                    row["_filename"] = filename
+                rows.append(row)
+        except Exception:
+            continue
+
+    if not rows:
+        df = _empty_df()
+        if include_filename:
+            df["_filename"] = []
+        return df
+
+    df = pd.DataFrame(rows)
+    for col in ["submitted_at", "username", "model_name", "model_api", "notes"]:
+        if col not in df.columns:
+            df[col] = ""
+    cols = ["submitted_at", "username", "model_name", "model_api", "notes"]
+    if include_filename:
+        if "_filename" not in df.columns:
+            df["_filename"] = ""
+        cols = cols + ["_filename"]
+    df = df[cols]
+    df = df.sort_values(by=["submitted_at"], ascending=False, kind="stable")
+    return df
+
+
+def refresh():
+    return _load_entries_df(APPROVED_PREFIX, include_filename=False)
+
+
+def refresh_pending():
+    df = _load_entries_df(PENDING_PREFIX, include_filename=True)
+    choices = list(df["_filename"]) if "_filename" in df.columns else []
+    return df, gr.update(choices=choices, value=choices[0] if choices else None)
+
+
+def submit(model_name: str, model_api: str, notes: str, username: str | None):
+    model_name = (model_name or "").strip()
+    model_api = (model_api or "").strip()
+    notes = (notes or "").strip()
+    username = (username or "").strip() or ("local" if not IN_SPACES else "anonymous")
+
+    if not model_name:
+        return "请填写 **模型名称**。", refresh()
+    if not model_api:
+        return "请填写 **模型 API**。", refresh()
+    if not _is_valid_http_url(model_api):
+        return "**模型 API** 需要是合法的 `http(s)://...` URL。", refresh()
+
+    if not HF_TOKEN:
+        return "Space 未配置 **HF_TOKEN**（Secrets），无法写入排行榜。", refresh()
+    if not LEADERBOARD_DATASET:
+        return "Space 未配置 **LEADERBOARD_DATASET**（例如：`your-username/miniapp-leaderboard`）。", refresh()
+
+    _ensure_dataset_repo()
+    api = _api()
+
+    now = datetime.datetime.utcnow().replace(microsecond=0).isoformat() + "Z"
+    safe_model = _slug(model_name)
+    safe_user = _slug(username)
+    nonce = uuid.uuid4().hex[:8]
+    path_in_repo = f"{PENDING_PREFIX}{now[:10]}/{now}-{safe_user}-{safe_model}-{nonce}.json"
+
+    payload = {
+        "submitted_at": now,
+        "username": username,
+        "model_name": model_name,
+        "model_api": model_api,
+        "notes": notes,
+    }
+    data = (json.dumps(payload, ensure_ascii=False, indent=2) + "\n").encode("utf-8")
+    bio = io.BytesIO(data)
+
+    api.upload_file(
+        repo_id=LEADERBOARD_DATASET,
+        repo_type="dataset",
+        path_or_fileobj=bio,
+        path_in_repo=path_in_repo,
+        commit_message=f"miniapp: submit(pending) {username}/{model_name}",
+        token=HF_TOKEN,
+    )
+
+    return "已提交，等待 owner 审核后才会上榜。", refresh()
+
+
+def approve(pending_filename: str | None, review_token: str | None):
+    pending_filename = (pending_filename or "").strip()
+    review_token = (review_token or "").strip()
+
+    if not pending_filename:
+        df, dd = refresh_pending()
+        return "请选择一条待审核提交。", df, dd, refresh()
+
+    if not OWNER_REVIEW_TOKEN:
+        df, dd = refresh_pending()
+        return "Space 未配置 **OWNER_REVIEW_TOKEN**（Secrets），无法启用审核。", df, dd, refresh()
+
+    if review_token != OWNER_REVIEW_TOKEN:
+        df, dd = refresh_pending()
+        return "审核口令不正确。", df, dd, refresh()
+
+    if not HF_TOKEN or not LEADERBOARD_DATASET:
+        df, dd = refresh_pending()
+        return "Space 未配置 HF 写入权限，无法审核。", df, dd, refresh()
+
+    api = _api()
+    try:
+        local_path = hf_hub_download(
+            repo_id=LEADERBOARD_DATASET,
+            repo_type="dataset",
+            filename=pending_filename,
+            token=HF_TOKEN,
+        )
+        with open(local_path, "r", encoding="utf-8") as fp:
+            payload = json.load(fp)
+    except Exception as e:
+        df, dd = refresh_pending()
+        return f"读取待审核文件失败：{e}", df, dd, refresh()
+
+    # 将文件复制到 approved 目录（保留原提交时间等字段）
+    base = pending_filename[len(PENDING_PREFIX) :] if pending_filename.startswith(PENDING_PREFIX) else pending_filename
+    approved_filename = f"{APPROVED_PREFIX}{base}"
+    data = (json.dumps(payload, ensure_ascii=False, indent=2) + "\n").encode("utf-8")
+    bio = io.BytesIO(data)
+
+    try:
+        api.upload_file(
+            repo_id=LEADERBOARD_DATASET,
+            repo_type="dataset",
+            path_or_fileobj=bio,
+            path_in_repo=approved_filename,
+            commit_message=f"miniapp: approve {payload.get('username','')}/{payload.get('model_name','')}",
+            token=HF_TOKEN,
+        )
+        # 删除 pending 原文件（真正“上榜前审核”）
+        api.delete_file(
+            repo_id=LEADERBOARD_DATASET,
+            repo_type="dataset",
+            path_in_repo=pending_filename,
+            commit_message=f"miniapp: remove pending {pending_filename}",
+            token=HF_TOKEN,
+        )
+    except Exception as e:
+        df, dd = refresh_pending()
+        return f"审核写入失败：{e}", df, dd, refresh()
+
+    pending_df, pending_dd = refresh_pending()
+    approved_df = refresh()
+    return "已通过审核并更新榜单。", pending_df, pending_dd, approved_df
+
+
+with gr.Blocks(title=f"{APP_NAME} leaderboard") as demo:
+    gr.Markdown(
+        f"## {APP_NAME} leaderboard\n\n"
+        "用户提交信息后会进入 **pending（待审核）**；owner 审核通过后才会进入 **approved（上榜）**。\n\n"
+        f"- 当前 `LEADERBOARD_DATASET`: `{LEADERBOARD_DATASET or '(未配置)'}`\n"
+    )
+
+    with gr.Row():
+        with gr.Column(scale=2):
+            model_name = gr.Textbox(label="模型名称（必填）", placeholder="例如：my-agent-v1")
+            model_api = gr.Textbox(
+                label="模型 API（必填）",
+                placeholder="例如：https://api.example.com/v1/chat/completions",
+            )
+            notes = gr.Textbox(label="备注（可选）", lines=4)
+            username = gr.Textbox(
+                label="用户名（可选）",
+                placeholder="建议填你的 HF 用户名（也可留空）",
+                value="" if IN_SPACES else "local",
+            )
+            submit_btn = gr.Button("提交", variant="primary")
+            status = gr.Markdown()
+        with gr.Column(scale=3):
+            leaderboard = gr.Dataframe(
+                label="Leaderboard（按提交时间倒序）",
+                value=_load_entries_df(APPROVED_PREFIX, include_filename=False),
+                interactive=False,
+                wrap=True,
+            )
+            refresh_btn = gr.Button("刷新")
+
+    submit_btn.click(
+        submit,
+        inputs=[model_name, model_api, notes, username],
+        outputs=[status, leaderboard],
+    )
+    refresh_btn.click(refresh, inputs=[], outputs=[leaderboard])
+
+    with gr.Accordion("Owner 审核（测试 demo）", open=False):
+        gr.Markdown(
+            "需要在 Space Secrets 中配置 `OWNER_REVIEW_TOKEN`。只有输入正确口令，才允许把 pending 放入榜单。"
+        )
+        review_token = gr.Textbox(label="审核口令", type="password", placeholder="OWNER_REVIEW_TOKEN")
+        pending_refresh_btn = gr.Button("刷新待审核列表")
+        pending_df = gr.Dataframe(
+            label="待审核（pending）",
+            value=_load_entries_df(PENDING_PREFIX, include_filename=True),
+            interactive=False,
+            wrap=True,
+        )
+        pending_pick = gr.Dropdown(
+            label="选择要通过的提交（文件）",
+            choices=_list_json_files(PENDING_PREFIX),
+        )
+        approve_btn = gr.Button("通过审核并上榜", variant="primary")
+        approve_status = gr.Markdown()
+
+        pending_refresh_btn.click(
+            refresh_pending,
+            inputs=[],
+            outputs=[pending_df, pending_pick],
+        )
+        approve_btn.click(
+            approve,
+            inputs=[pending_pick, review_token],
+            outputs=[approve_status, pending_df, pending_pick, leaderboard],
+        )
+
+demo.launch()
+

requirements.txt

@@ -0,0 +1,4 @@
+gradio
+huggingface-hub
+pandas
+requests