submission.py

from __future__ import annotations

import html
import json
import math
import os
import re
import urllib.error
import urllib.request
from datetime import datetime, timezone
from pathlib import Path
from typing import Any

import pandas as pd

from leaderboard import compute_total_from_metrics


SUBMISSION_METRICS = [
    "Vis",
    "Aud (PQ)",
    "AV",
    "Lip",
    "Text",
    "Face",
    "Music",
    "Speech",
    "Lo-Phy",
    "Hi-Phy",
    "Holistic",
]

COMPONENT_TYPES = ["Proprietary", "Open-source", "Mixed"]

METRIC_RANGES = {
    "Vis": (0.0, 1.0),
    "Aud (PQ)": (0.0, 10.0),
    "AV": (0.0, 10.0),
    "Lip": (0.0, 100.0),
    "Text": (0.0, 100.0),
    "Face": (0.0, 100.0),
    "Music": (0.0, 100.0),
    "Speech": (0.0, 100.0),
    "Lo-Phy": (0.0, 5.0),
    "Hi-Phy": (0.0, 100.0),
    "Holistic": (0.0, 100.0),
}


class SubmissionError(ValueError):
    pass


def submit_score(
    model: str,
    components: str,
    component_type: str,
    contact: str,
    model_url: str,
    results_url: str,
    notes: str,
    vis: float,
    aud_pq: float,
    av: float,
    lip: float,
    text: float,
    face: float,
    music: float,
    speech: float,
    lo_phy: float,
    hi_phy: float,
    holistic: float,
) -> tuple[str, str]:
    try:
        submission = build_submission(
            model=model,
            components=components,
            component_type=component_type,
            contact=contact,
            model_url=model_url,
            results_url=results_url,
            notes=notes,
            metrics={
                "Vis": vis,
                "Aud (PQ)": aud_pq,
                "AV": av,
                "Lip": lip,
                "Text": text,
                "Face": face,
                "Music": music,
                "Speech": speech,
                "Lo-Phy": lo_phy,
                "Hi-Phy": hi_phy,
                "Holistic": holistic,
            },
        )
        destination = persist_submission(submission)
        status_html = render_submission_status(submission, destination)
        return status_html, submission_to_json(submission)
    except SubmissionError as exc:
        return render_error_status(str(exc)), ""
    except Exception as exc:
        return render_error_status(f"Submission failed: {exc}"), ""


def build_submission(
    *,
    model: str,
    components: str,
    component_type: str,
    contact: str,
    model_url: str,
    results_url: str,
    notes: str,
    metrics: dict[str, Any],
) -> dict[str, Any]:
    model = _clean_required(model, "Model name", min_len=2, max_len=120)
    components = _clean_required(components, "Components", min_len=2, max_len=240)
    contact = _clean_required(contact, "Public contact", min_len=2, max_len=160)
    model_url = _clean_required(model_url, "Model or paper URL", min_len=8, max_len=500)
    results_url = _clean_required(results_url, "Evaluation artifact URL", min_len=8, max_len=500)
    notes = (notes or "").strip()
    if len(notes) > 2000:
        raise SubmissionError("Notes must be 2000 characters or fewer.")
    if component_type not in COMPONENT_TYPES:
        raise SubmissionError("Component type must be Proprietary, Open-source, or Mixed.")
    _validate_url(model_url, "Model or paper URL")
    _validate_url(results_url, "Evaluation artifact URL")

    metric_values = {metric: _validate_metric(metric, metrics.get(metric)) for metric in SUBMISSION_METRICS}
    total = compute_total_from_metrics(pd.Series(metric_values))
    if pd.isna(total):
        raise SubmissionError("Could not compute Total from the submitted metrics.")

    return {
        "schema_version": 1,
        "status": "pending_review",
        "submitted_at_utc": datetime.now(timezone.utc).isoformat(timespec="seconds"),
        "model": model,
        "components": components,
        "component_type": component_type,
        "contact": contact,
        "model_url": model_url,
        "evaluation_artifact_url": results_url,
        "notes": notes,
        "metrics": metric_values,
        "computed_total": round(float(total), 4),
    }


def persist_submission(submission: dict[str, Any]) -> dict[str, str]:
    backend = os.environ.get("SUBMISSION_BACKEND", "").strip().lower()
    token = os.environ.get("GITHUB_TOKEN", "").strip()
    repo = os.environ.get("GITHUB_REPO", "").strip()

    if not backend:
        backend = "github_issue" if token and repo else "local_file"

    if backend == "disabled":
        raise SubmissionError("Public submission is currently disabled.")
    if backend == "github_issue":
        if not token or not repo:
            raise SubmissionError("GITHUB_TOKEN and GITHUB_REPO are required for github_issue backend.")
        issue_url = create_github_issue(submission, repo, token)
        return {"backend": "github_issue", "url": issue_url}
    if backend == "local_file":
        path = save_submission_file(submission)
        return {"backend": "local_file", "path": str(path)}

    raise SubmissionError(f"Unknown SUBMISSION_BACKEND: {backend}")


def create_github_issue(submission: dict[str, Any], repo: str, token: str) -> str:
    payload = {
        "title": f"[Leaderboard Submission] {submission['model']}",
        "body": github_issue_body(submission),
        "labels": ["leaderboard-submission", "needs-review"],
    }
    request = urllib.request.Request(
        url=f"https://api.github.com/repos/{repo}/issues",
        data=json.dumps(payload).encode("utf-8"),
        headers={
            "Accept": "application/vnd.github+json",
            "Authorization": f"Bearer {token}",
            "Content-Type": "application/json",
            "User-Agent": "AVGen-Bench-Leaderboard",
            "X-GitHub-Api-Version": "2022-11-28",
        },
        method="POST",
    )
    try:
        with urllib.request.urlopen(request, timeout=20) as response:
            data = json.loads(response.read().decode("utf-8"))
    except urllib.error.HTTPError as exc:
        detail = exc.read().decode("utf-8", errors="replace")
        raise SubmissionError(f"GitHub issue creation failed ({exc.code}): {detail}") from exc
    except urllib.error.URLError as exc:
        raise SubmissionError(f"Could not reach GitHub API: {exc.reason}") from exc

    issue_url = data.get("html_url")
    if not issue_url:
        raise SubmissionError("GitHub API response did not include an issue URL.")
    return str(issue_url)


def save_submission_file(submission: dict[str, Any]) -> Path:
    root = Path(os.environ.get("PENDING_SUBMISSION_DIR", "pending_submissions"))
    root.mkdir(parents=True, exist_ok=True)
    filename = f"{_slugify(submission['model'])}-{_compact_timestamp(submission['submitted_at_utc'])}.json"
    path = root / filename
    path.write_text(submission_to_json(submission) + "\n", encoding="utf-8")
    return path


def github_issue_body(submission: dict[str, Any]) -> str:
    metrics = submission["metrics"]
    metric_rows = "\n".join(f"| {metric} | {metrics[metric]} |" for metric in SUBMISSION_METRICS)
    return f"""## Submission

| Field | Value |
|---|---|
| Model | {submission['model']} |
| Components | {submission['components']} |
| Component Type | {submission['component_type']} |
| Computed Total | {submission['computed_total']:.2f} |
| Contact | {submission['contact']} |
| Model or Paper URL | {submission['model_url']} |
| Evaluation Artifact URL | {submission['evaluation_artifact_url']} |
| Submitted At UTC | {submission['submitted_at_utc']} |

## Raw Metrics

| Metric | Value |
|---|---:|
{metric_rows}

## Notes

{submission['notes'] or 'None'}

## Machine-Readable Payload

```json
{submission_to_json(submission)}
```
"""


def submission_to_json(submission: dict[str, Any]) -> str:
    return json.dumps(submission, ensure_ascii=False, indent=2, sort_keys=True)


def render_submission_status(submission: dict[str, Any], destination: dict[str, str]) -> str:
    if destination["backend"] == "github_issue":
        detail = f'<a href="{html.escape(destination["url"])}" target="_blank" rel="noopener">GitHub issue</a>'
    else:
        detail = html.escape(destination["path"])
    return f"""
    <div class="status-card success">
      <strong>Submission received for review.</strong>
      <p>Total was recomputed from raw metrics: <b>{submission['computed_total']:.2f}</b>.</p>
      <p>Destination: {detail}</p>
    </div>
    """


def render_error_status(message: str) -> str:
    return f"""
    <div class="status-card error">
      <strong>Submission not accepted.</strong>
      <p>{html.escape(message)}</p>
    </div>
    """


def _clean_required(value: str, label: str, *, min_len: int, max_len: int) -> str:
    value = (value or "").strip()
    if len(value) < min_len:
        raise SubmissionError(f"{label} is required.")
    if len(value) > max_len:
        raise SubmissionError(f"{label} must be {max_len} characters or fewer.")
    return value


def _validate_url(value: str, label: str) -> None:
    if not re.match(r"^https?://", value):
        raise SubmissionError(f"{label} must start with http:// or https://.")


def _validate_metric(metric: str, value: Any) -> float:
    if value is None or value == "":
        raise SubmissionError(f"{metric} is required.")
    try:
        numeric = float(value)
    except (TypeError, ValueError) as exc:
        raise SubmissionError(f"{metric} must be numeric.") from exc
    if not math.isfinite(numeric):
        raise SubmissionError(f"{metric} must be finite.")
    low, high = METRIC_RANGES[metric]
    if numeric < low or numeric > high:
        raise SubmissionError(f"{metric} must be between {low:g} and {high:g}.")
    return round(numeric, 6)


def _slugify(value: str) -> str:
    slug = re.sub(r"[^a-zA-Z0-9]+", "-", value.strip().lower()).strip("-")
    return slug or "submission"


def _compact_timestamp(value: str) -> str:
    return re.sub(r"[^0-9]", "", value)[:14]